drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_cycles.h>
  28 #include <rte_ip.h>
  29 #include <rte_gre.h>
  30 #include <rte_vxlan.h>
  31 #include <rte_gtp.h>
  32 #include <rte_eal_paging.h>
  33
  34 #include <mlx5_devx_cmds.h>
  35 #include <mlx5_prm.h>
  36 #include <mlx5_malloc.h>
  37
  38 #include "mlx5_defs.h"
  39 #include "mlx5.h"
  40 #include "mlx5_common_os.h"
  41 #include "mlx5_flow.h"
  42 #include "mlx5_flow_os.h"
  43 #include "mlx5_rxtx.h"
  44
  45 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  46
  47 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  48 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  49 #endif
  50
  51 #ifndef HAVE_MLX5DV_DR_ESWITCH
  52 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  53 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  54 #endif
  55 #endif
  56
  57 #ifndef HAVE_MLX5DV_DR
  58 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  59 #endif
  60
  61 /* VLAN header definitions */
  62 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  65 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  66 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  67
  68 union flow_dv_attr {
  69         struct {
  70                 uint32_t valid:1;
  71                 uint32_t ipv4:1;
  72                 uint32_t ipv6:1;
  73                 uint32_t tcp:1;
  74                 uint32_t udp:1;
  75                 uint32_t reserved:27;
  76         };
  77         uint32_t attr;
  78 };
  79
  80 static int
  81 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  82                              struct mlx5_flow_tbl_resource *tbl);
  83
  84 static int
  85 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NONE] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NONE);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NONE);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1164                 {0, 0, 0},
1165         };
1166         int reg;
1167
1168         if (!mask)
1169                 return rte_flow_error_set(error, EINVAL,
1170                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1171                                           NULL, "zero mark action mask");
1172         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1173         if (reg < 0)
1174                 return reg;
1175         MLX5_ASSERT(reg > 0);
1176         if (reg == REG_C_0) {
1177                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1178                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1179
1180                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1181                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1182                 mask = rte_cpu_to_be_32(mask << shl_c0);
1183         }
1184         reg_c_x[0].id = reg_to_field[reg];
1185         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1186                                              MLX5_MODIFICATION_TYPE_SET, error);
1187 }
1188
1189 /**
1190  * Get metadata register index for specified steering domain.
1191  *
1192  * @param[in] dev
1193  *   Pointer to the rte_eth_dev structure.
1194  * @param[in] attr
1195  *   Attributes of flow to determine steering domain.
1196  * @param[out] error
1197  *   Pointer to the error structure.
1198  *
1199  * @return
1200  *   positive index on success, a negative errno value otherwise
1201  *   and rte_errno is set.
1202  */
1203 static enum modify_reg
1204 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1205                          const struct rte_flow_attr *attr,
1206                          struct rte_flow_error *error)
1207 {
1208         int reg =
1209                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1210                                           MLX5_METADATA_FDB :
1211                                             attr->egress ?
1212                                             MLX5_METADATA_TX :
1213                                             MLX5_METADATA_RX, 0, error);
1214         if (reg < 0)
1215                 return rte_flow_error_set(error,
1216                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1217                                           NULL, "unavailable "
1218                                           "metadata register");
1219         return reg;
1220 }
1221
1222 /**
1223  * Convert SET_META action to DV specification.
1224  *
1225  * @param[in] dev
1226  *   Pointer to the rte_eth_dev structure.
1227  * @param[in,out] resource
1228  *   Pointer to the modify-header resource.
1229  * @param[in] attr
1230  *   Attributes of flow that includes this item.
1231  * @param[in] conf
1232  *   Pointer to action specification.
1233  * @param[out] error
1234  *   Pointer to the error structure.
1235  *
1236  * @return
1237  *   0 on success, a negative errno value otherwise and rte_errno is set.
1238  */
1239 static int
1240 flow_dv_convert_action_set_meta
1241                         (struct rte_eth_dev *dev,
1242                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1243                          const struct rte_flow_attr *attr,
1244                          const struct rte_flow_action_set_meta *conf,
1245                          struct rte_flow_error *error)
1246 {
1247         uint32_t data = conf->data;
1248         uint32_t mask = conf->mask;
1249         struct rte_flow_item item = {
1250                 .spec = &data,
1251                 .mask = &mask,
1252         };
1253         struct field_modify_info reg_c_x[] = {
1254                 [1] = {0, 0, 0},
1255         };
1256         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1257
1258         if (reg < 0)
1259                 return reg;
1260         /*
1261          * In datapath code there is no endianness
1262          * coversions for perfromance reasons, all
1263          * pattern conversions are done in rte_flow.
1264          */
1265         if (reg == REG_C_0) {
1266                 struct mlx5_priv *priv = dev->data->dev_private;
1267                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1268                 uint32_t shl_c0;
1269
1270                 MLX5_ASSERT(msk_c0);
1271 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1272                 shl_c0 = rte_bsf32(msk_c0);
1273 #else
1274                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1275 #endif
1276                 mask <<= shl_c0;
1277                 data <<= shl_c0;
1278                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1279         }
1280         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1281         /* The routine expects parameters in memory as big-endian ones. */
1282         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1283                                              MLX5_MODIFICATION_TYPE_SET, error);
1284 }
1285
1286 /**
1287  * Convert modify-header set IPv4 DSCP action to DV specification.
1288  *
1289  * @param[in,out] resource
1290  *   Pointer to the modify-header resource.
1291  * @param[in] action
1292  *   Pointer to action specification.
1293  * @param[out] error
1294  *   Pointer to the error structure.
1295  *
1296  * @return
1297  *   0 on success, a negative errno value otherwise and rte_errno is set.
1298  */
1299 static int
1300 flow_dv_convert_action_modify_ipv4_dscp
1301                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1302                          const struct rte_flow_action *action,
1303                          struct rte_flow_error *error)
1304 {
1305         const struct rte_flow_action_set_dscp *conf =
1306                 (const struct rte_flow_action_set_dscp *)(action->conf);
1307         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1308         struct rte_flow_item_ipv4 ipv4;
1309         struct rte_flow_item_ipv4 ipv4_mask;
1310
1311         memset(&ipv4, 0, sizeof(ipv4));
1312         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1313         ipv4.hdr.type_of_service = conf->dscp;
1314         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1315         item.spec = &ipv4;
1316         item.mask = &ipv4_mask;
1317         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1318                                              MLX5_MODIFICATION_TYPE_SET, error);
1319 }
1320
1321 /**
1322  * Convert modify-header set IPv6 DSCP action to DV specification.
1323  *
1324  * @param[in,out] resource
1325  *   Pointer to the modify-header resource.
1326  * @param[in] action
1327  *   Pointer to action specification.
1328  * @param[out] error
1329  *   Pointer to the error structure.
1330  *
1331  * @return
1332  *   0 on success, a negative errno value otherwise and rte_errno is set.
1333  */
1334 static int
1335 flow_dv_convert_action_modify_ipv6_dscp
1336                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1337                          const struct rte_flow_action *action,
1338                          struct rte_flow_error *error)
1339 {
1340         const struct rte_flow_action_set_dscp *conf =
1341                 (const struct rte_flow_action_set_dscp *)(action->conf);
1342         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1343         struct rte_flow_item_ipv6 ipv6;
1344         struct rte_flow_item_ipv6 ipv6_mask;
1345
1346         memset(&ipv6, 0, sizeof(ipv6));
1347         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1348         /*
1349          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1350          * rdma-core only accept the DSCP bits byte aligned start from
1351          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1352          * bits in IPv6 case as rdma-core requires byte aligned value.
1353          */
1354         ipv6.hdr.vtc_flow = conf->dscp;
1355         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1356         item.spec = &ipv6;
1357         item.mask = &ipv6_mask;
1358         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1359                                              MLX5_MODIFICATION_TYPE_SET, error);
1360 }
1361
1362 /**
1363  * Validate MARK item.
1364  *
1365  * @param[in] dev
1366  *   Pointer to the rte_eth_dev structure.
1367  * @param[in] item
1368  *   Item specification.
1369  * @param[in] attr
1370  *   Attributes of flow that includes this item.
1371  * @param[out] error
1372  *   Pointer to error structure.
1373  *
1374  * @return
1375  *   0 on success, a negative errno value otherwise and rte_errno is set.
1376  */
1377 static int
1378 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1379                            const struct rte_flow_item *item,
1380                            const struct rte_flow_attr *attr __rte_unused,
1381                            struct rte_flow_error *error)
1382 {
1383         struct mlx5_priv *priv = dev->data->dev_private;
1384         struct mlx5_dev_config *config = &priv->config;
1385         const struct rte_flow_item_mark *spec = item->spec;
1386         const struct rte_flow_item_mark *mask = item->mask;
1387         const struct rte_flow_item_mark nic_mask = {
1388                 .id = priv->sh->dv_mark_mask,
1389         };
1390         int ret;
1391
1392         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1393                 return rte_flow_error_set(error, ENOTSUP,
1394                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1395                                           "extended metadata feature"
1396                                           " isn't enabled");
1397         if (!mlx5_flow_ext_mreg_supported(dev))
1398                 return rte_flow_error_set(error, ENOTSUP,
1399                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1400                                           "extended metadata register"
1401                                           " isn't supported");
1402         if (!nic_mask.id)
1403                 return rte_flow_error_set(error, ENOTSUP,
1404                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1405                                           "extended metadata register"
1406                                           " isn't available");
1407         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1408         if (ret < 0)
1409                 return ret;
1410         if (!spec)
1411                 return rte_flow_error_set(error, EINVAL,
1412                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1413                                           item->spec,
1414                                           "data cannot be empty");
1415         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1416                 return rte_flow_error_set(error, EINVAL,
1417                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1418                                           &spec->id,
1419                                           "mark id exceeds the limit");
1420         if (!mask)
1421                 mask = &nic_mask;
1422         if (!mask->id)
1423                 return rte_flow_error_set(error, EINVAL,
1424                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1425                                         "mask cannot be zero");
1426
1427         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1428                                         (const uint8_t *)&nic_mask,
1429                                         sizeof(struct rte_flow_item_mark),
1430                                         error);
1431         if (ret < 0)
1432                 return ret;
1433         return 0;
1434 }
1435
1436 /**
1437  * Validate META item.
1438  *
1439  * @param[in] dev
1440  *   Pointer to the rte_eth_dev structure.
1441  * @param[in] item
1442  *   Item specification.
1443  * @param[in] attr
1444  *   Attributes of flow that includes this item.
1445  * @param[out] error
1446  *   Pointer to error structure.
1447  *
1448  * @return
1449  *   0 on success, a negative errno value otherwise and rte_errno is set.
1450  */
1451 static int
1452 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1453                            const struct rte_flow_item *item,
1454                            const struct rte_flow_attr *attr,
1455                            struct rte_flow_error *error)
1456 {
1457         struct mlx5_priv *priv = dev->data->dev_private;
1458         struct mlx5_dev_config *config = &priv->config;
1459         const struct rte_flow_item_meta *spec = item->spec;
1460         const struct rte_flow_item_meta *mask = item->mask;
1461         struct rte_flow_item_meta nic_mask = {
1462                 .data = UINT32_MAX
1463         };
1464         int reg;
1465         int ret;
1466
1467         if (!spec)
1468                 return rte_flow_error_set(error, EINVAL,
1469                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1470                                           item->spec,
1471                                           "data cannot be empty");
1472         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1473                 if (!mlx5_flow_ext_mreg_supported(dev))
1474                         return rte_flow_error_set(error, ENOTSUP,
1475                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1476                                           "extended metadata register"
1477                                           " isn't supported");
1478                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1479                 if (reg < 0)
1480                         return reg;
1481                 if (reg == REG_B)
1482                         return rte_flow_error_set(error, ENOTSUP,
1483                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1484                                           "match on reg_b "
1485                                           "isn't supported");
1486                 if (reg != REG_A)
1487                         nic_mask.data = priv->sh->dv_meta_mask;
1488         } else if (attr->transfer) {
1489                 return rte_flow_error_set(error, ENOTSUP,
1490                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1491                                         "extended metadata feature "
1492                                         "should be enabled when "
1493                                         "meta item is requested "
1494                                         "with e-switch mode ");
1495         }
1496         if (!mask)
1497                 mask = &rte_flow_item_meta_mask;
1498         if (!mask->data)
1499                 return rte_flow_error_set(error, EINVAL,
1500                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1501                                         "mask cannot be zero");
1502
1503         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1504                                         (const uint8_t *)&nic_mask,
1505                                         sizeof(struct rte_flow_item_meta),
1506                                         error);
1507         return ret;
1508 }
1509
1510 /**
1511  * Validate TAG item.
1512  *
1513  * @param[in] dev
1514  *   Pointer to the rte_eth_dev structure.
1515  * @param[in] item
1516  *   Item specification.
1517  * @param[in] attr
1518  *   Attributes of flow that includes this item.
1519  * @param[out] error
1520  *   Pointer to error structure.
1521  *
1522  * @return
1523  *   0 on success, a negative errno value otherwise and rte_errno is set.
1524  */
1525 static int
1526 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1527                           const struct rte_flow_item *item,
1528                           const struct rte_flow_attr *attr __rte_unused,
1529                           struct rte_flow_error *error)
1530 {
1531         const struct rte_flow_item_tag *spec = item->spec;
1532         const struct rte_flow_item_tag *mask = item->mask;
1533         const struct rte_flow_item_tag nic_mask = {
1534                 .data = RTE_BE32(UINT32_MAX),
1535                 .index = 0xff,
1536         };
1537         int ret;
1538
1539         if (!mlx5_flow_ext_mreg_supported(dev))
1540                 return rte_flow_error_set(error, ENOTSUP,
1541                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1542                                           "extensive metadata register"
1543                                           " isn't supported");
1544         if (!spec)
1545                 return rte_flow_error_set(error, EINVAL,
1546                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1547                                           item->spec,
1548                                           "data cannot be empty");
1549         if (!mask)
1550                 mask = &rte_flow_item_tag_mask;
1551         if (!mask->data)
1552                 return rte_flow_error_set(error, EINVAL,
1553                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1554                                         "mask cannot be zero");
1555
1556         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1557                                         (const uint8_t *)&nic_mask,
1558                                         sizeof(struct rte_flow_item_tag),
1559                                         error);
1560         if (ret < 0)
1561                 return ret;
1562         if (mask->index != 0xff)
1563                 return rte_flow_error_set(error, EINVAL,
1564                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1565                                           "partial mask for tag index"
1566                                           " is not supported");
1567         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1568         if (ret < 0)
1569                 return ret;
1570         MLX5_ASSERT(ret != REG_NONE);
1571         return 0;
1572 }
1573
1574 /**
1575  * Validate vport item.
1576  *
1577  * @param[in] dev
1578  *   Pointer to the rte_eth_dev structure.
1579  * @param[in] item
1580  *   Item specification.
1581  * @param[in] attr
1582  *   Attributes of flow that includes this item.
1583  * @param[in] item_flags
1584  *   Bit-fields that holds the items detected until now.
1585  * @param[out] error
1586  *   Pointer to error structure.
1587  *
1588  * @return
1589  *   0 on success, a negative errno value otherwise and rte_errno is set.
1590  */
1591 static int
1592 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1593                               const struct rte_flow_item *item,
1594                               const struct rte_flow_attr *attr,
1595                               uint64_t item_flags,
1596                               struct rte_flow_error *error)
1597 {
1598         const struct rte_flow_item_port_id *spec = item->spec;
1599         const struct rte_flow_item_port_id *mask = item->mask;
1600         const struct rte_flow_item_port_id switch_mask = {
1601                         .id = 0xffffffff,
1602         };
1603         struct mlx5_priv *esw_priv;
1604         struct mlx5_priv *dev_priv;
1605         int ret;
1606
1607         if (!attr->transfer)
1608                 return rte_flow_error_set(error, EINVAL,
1609                                           RTE_FLOW_ERROR_TYPE_ITEM,
1610                                           NULL,
1611                                           "match on port id is valid only"
1612                                           " when transfer flag is enabled");
1613         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1614                 return rte_flow_error_set(error, ENOTSUP,
1615                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1616                                           "multiple source ports are not"
1617                                           " supported");
1618         if (!mask)
1619                 mask = &switch_mask;
1620         if (mask->id != 0xffffffff)
1621                 return rte_flow_error_set(error, ENOTSUP,
1622                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1623                                            mask,
1624                                            "no support for partial mask on"
1625                                            " \"id\" field");
1626         ret = mlx5_flow_item_acceptable
1627                                 (item, (const uint8_t *)mask,
1628                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1629                                  sizeof(struct rte_flow_item_port_id),
1630                                  error);
1631         if (ret)
1632                 return ret;
1633         if (!spec)
1634                 return 0;
1635         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1636         if (!esw_priv)
1637                 return rte_flow_error_set(error, rte_errno,
1638                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1639                                           "failed to obtain E-Switch info for"
1640                                           " port");
1641         dev_priv = mlx5_dev_to_eswitch_info(dev);
1642         if (!dev_priv)
1643                 return rte_flow_error_set(error, rte_errno,
1644                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1645                                           NULL,
1646                                           "failed to obtain E-Switch info");
1647         if (esw_priv->domain_id != dev_priv->domain_id)
1648                 return rte_flow_error_set(error, EINVAL,
1649                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1650                                           "cannot match on a port from a"
1651                                           " different E-Switch");
1652         return 0;
1653 }
1654
1655 /**
1656  * Validate VLAN item.
1657  *
1658  * @param[in] item
1659  *   Item specification.
1660  * @param[in] item_flags
1661  *   Bit-fields that holds the items detected until now.
1662  * @param[in] dev
1663  *   Ethernet device flow is being created on.
1664  * @param[out] error
1665  *   Pointer to error structure.
1666  *
1667  * @return
1668  *   0 on success, a negative errno value otherwise and rte_errno is set.
1669  */
1670 static int
1671 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1672                            uint64_t item_flags,
1673                            struct rte_eth_dev *dev,
1674                            struct rte_flow_error *error)
1675 {
1676         const struct rte_flow_item_vlan *mask = item->mask;
1677         const struct rte_flow_item_vlan nic_mask = {
1678                 .tci = RTE_BE16(UINT16_MAX),
1679                 .inner_type = RTE_BE16(UINT16_MAX),
1680         };
1681         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1682         int ret;
1683         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1684                                         MLX5_FLOW_LAYER_INNER_L4) :
1685                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1686                                         MLX5_FLOW_LAYER_OUTER_L4);
1687         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1688                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1689
1690         if (item_flags & vlanm)
1691                 return rte_flow_error_set(error, EINVAL,
1692                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1693                                           "multiple VLAN layers not supported");
1694         else if ((item_flags & l34m) != 0)
1695                 return rte_flow_error_set(error, EINVAL,
1696                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1697                                           "VLAN cannot follow L3/L4 layer");
1698         if (!mask)
1699                 mask = &rte_flow_item_vlan_mask;
1700         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1701                                         (const uint8_t *)&nic_mask,
1702                                         sizeof(struct rte_flow_item_vlan),
1703                                         error);
1704         if (ret)
1705                 return ret;
1706         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1707                 struct mlx5_priv *priv = dev->data->dev_private;
1708
1709                 if (priv->vmwa_context) {
1710                         /*
1711                          * Non-NULL context means we have a virtual machine
1712                          * and SR-IOV enabled, we have to create VLAN interface
1713                          * to make hypervisor to setup E-Switch vport
1714                          * context correctly. We avoid creating the multiple
1715                          * VLAN interfaces, so we cannot support VLAN tag mask.
1716                          */
1717                         return rte_flow_error_set(error, EINVAL,
1718                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1719                                                   item,
1720                                                   "VLAN tag mask is not"
1721                                                   " supported in virtual"
1722                                                   " environment");
1723                 }
1724         }
1725         return 0;
1726 }
1727
1728 /*
1729  * GTP flags are contained in 1 byte of the format:
1730  * -------------------------------------------
1731  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1732  * |-----------------------------------------|
1733  * | value | Version | PT | Res | E | S | PN |
1734  * -------------------------------------------
1735  *
1736  * Matching is supported only for GTP flags E, S, PN.
1737  */
1738 #define MLX5_GTP_FLAGS_MASK     0x07
1739
1740 /**
1741  * Validate GTP item.
1742  *
1743  * @param[in] dev
1744  *   Pointer to the rte_eth_dev structure.
1745  * @param[in] item
1746  *   Item specification.
1747  * @param[in] item_flags
1748  *   Bit-fields that holds the items detected until now.
1749  * @param[out] error
1750  *   Pointer to error structure.
1751  *
1752  * @return
1753  *   0 on success, a negative errno value otherwise and rte_errno is set.
1754  */
1755 static int
1756 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1757                           const struct rte_flow_item *item,
1758                           uint64_t item_flags,
1759                           struct rte_flow_error *error)
1760 {
1761         struct mlx5_priv *priv = dev->data->dev_private;
1762         const struct rte_flow_item_gtp *spec = item->spec;
1763         const struct rte_flow_item_gtp *mask = item->mask;
1764         const struct rte_flow_item_gtp nic_mask = {
1765                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1766                 .msg_type = 0xff,
1767                 .teid = RTE_BE32(0xffffffff),
1768         };
1769
1770         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1771                 return rte_flow_error_set(error, ENOTSUP,
1772                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1773                                           "GTP support is not enabled");
1774         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1775                 return rte_flow_error_set(error, ENOTSUP,
1776                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1777                                           "multiple tunnel layers not"
1778                                           " supported");
1779         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1780                 return rte_flow_error_set(error, EINVAL,
1781                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1782                                           "no outer UDP layer found");
1783         if (!mask)
1784                 mask = &rte_flow_item_gtp_mask;
1785         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1786                 return rte_flow_error_set(error, ENOTSUP,
1787                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1788                                           "Match is supported for GTP"
1789                                           " flags only");
1790         return mlx5_flow_item_acceptable
1791                 (item, (const uint8_t *)mask,
1792                  (const uint8_t *)&nic_mask,
1793                  sizeof(struct rte_flow_item_gtp),
1794                  error);
1795 }
1796
1797 /**
1798  * Validate the pop VLAN action.
1799  *
1800  * @param[in] dev
1801  *   Pointer to the rte_eth_dev structure.
1802  * @param[in] action_flags
1803  *   Holds the actions detected until now.
1804  * @param[in] action
1805  *   Pointer to the pop vlan action.
1806  * @param[in] item_flags
1807  *   The items found in this flow rule.
1808  * @param[in] attr
1809  *   Pointer to flow attributes.
1810  * @param[out] error
1811  *   Pointer to error structure.
1812  *
1813  * @return
1814  *   0 on success, a negative errno value otherwise and rte_errno is set.
1815  */
1816 static int
1817 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1818                                  uint64_t action_flags,
1819                                  const struct rte_flow_action *action,
1820                                  uint64_t item_flags,
1821                                  const struct rte_flow_attr *attr,
1822                                  struct rte_flow_error *error)
1823 {
1824         const struct mlx5_priv *priv = dev->data->dev_private;
1825
1826         (void)action;
1827         (void)attr;
1828         if (!priv->sh->pop_vlan_action)
1829                 return rte_flow_error_set(error, ENOTSUP,
1830                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1831                                           NULL,
1832                                           "pop vlan action is not supported");
1833         if (attr->egress)
1834                 return rte_flow_error_set(error, ENOTSUP,
1835                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1836                                           NULL,
1837                                           "pop vlan action not supported for "
1838                                           "egress");
1839         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1840                 return rte_flow_error_set(error, ENOTSUP,
1841                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1842                                           "no support for multiple VLAN "
1843                                           "actions");
1844         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1845                 return rte_flow_error_set(error, ENOTSUP,
1846                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1847                                           NULL,
1848                                           "cannot pop vlan without a "
1849                                           "match on (outer) vlan in the flow");
1850         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1851                 return rte_flow_error_set(error, EINVAL,
1852                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1853                                           "wrong action order, port_id should "
1854                                           "be after pop VLAN action");
1855         if (!attr->transfer && priv->representor)
1856                 return rte_flow_error_set(error, ENOTSUP,
1857                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1858                                           "pop vlan action for VF representor "
1859                                           "not supported on NIC table");
1860         return 0;
1861 }
1862
1863 /**
1864  * Get VLAN default info from vlan match info.
1865  *
1866  * @param[in] items
1867  *   the list of item specifications.
1868  * @param[out] vlan
1869  *   pointer VLAN info to fill to.
1870  *
1871  * @return
1872  *   0 on success, a negative errno value otherwise and rte_errno is set.
1873  */
1874 static void
1875 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1876                                   struct rte_vlan_hdr *vlan)
1877 {
1878         const struct rte_flow_item_vlan nic_mask = {
1879                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1880                                 MLX5DV_FLOW_VLAN_VID_MASK),
1881                 .inner_type = RTE_BE16(0xffff),
1882         };
1883
1884         if (items == NULL)
1885                 return;
1886         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1887                 int type = items->type;
1888
1889                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1890                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1891                         break;
1892         }
1893         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1894                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1895                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1896
1897                 /* If VLAN item in pattern doesn't contain data, return here. */
1898                 if (!vlan_v)
1899                         return;
1900                 if (!vlan_m)
1901                         vlan_m = &nic_mask;
1902                 /* Only full match values are accepted */
1903                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1904                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1905                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1906                         vlan->vlan_tci |=
1907                                 rte_be_to_cpu_16(vlan_v->tci &
1908                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1909                 }
1910                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1911                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1912                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1913                         vlan->vlan_tci |=
1914                                 rte_be_to_cpu_16(vlan_v->tci &
1915                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1916                 }
1917                 if (vlan_m->inner_type == nic_mask.inner_type)
1918                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1919                                                            vlan_m->inner_type);
1920         }
1921 }
1922
1923 /**
1924  * Validate the push VLAN action.
1925  *
1926  * @param[in] dev
1927  *   Pointer to the rte_eth_dev structure.
1928  * @param[in] action_flags
1929  *   Holds the actions detected until now.
1930  * @param[in] item_flags
1931  *   The items found in this flow rule.
1932  * @param[in] action
1933  *   Pointer to the action structure.
1934  * @param[in] attr
1935  *   Pointer to flow attributes
1936  * @param[out] error
1937  *   Pointer to error structure.
1938  *
1939  * @return
1940  *   0 on success, a negative errno value otherwise and rte_errno is set.
1941  */
1942 static int
1943 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1944                                   uint64_t action_flags,
1945                                   const struct rte_flow_item_vlan *vlan_m,
1946                                   const struct rte_flow_action *action,
1947                                   const struct rte_flow_attr *attr,
1948                                   struct rte_flow_error *error)
1949 {
1950         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1951         const struct mlx5_priv *priv = dev->data->dev_private;
1952
1953         if (!attr->transfer && attr->ingress)
1954                 return rte_flow_error_set(error, ENOTSUP,
1955                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1956                                           NULL,
1957                                           "push VLAN action not supported for "
1958                                           "ingress");
1959         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1960             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1961                 return rte_flow_error_set(error, EINVAL,
1962                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1963                                           "invalid vlan ethertype");
1964         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1965                 return rte_flow_error_set(error, ENOTSUP,
1966                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1967                                           "no support for multiple VLAN "
1968                                           "actions");
1969         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1970                 return rte_flow_error_set(error, EINVAL,
1971                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1972                                           "wrong action order, port_id should "
1973                                           "be after push VLAN");
1974         if (!attr->transfer && priv->representor)
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1977                                           "push vlan action for VF representor "
1978                                           "not supported on NIC table");
1979         if (vlan_m &&
1980             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1981             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1982                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1983             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1984             !(mlx5_flow_find_action
1985                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1986                 return rte_flow_error_set(error, EINVAL,
1987                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1988                                           "not full match mask on VLAN PCP and "
1989                                           "there is no of_set_vlan_pcp action, "
1990                                           "push VLAN action cannot figure out "
1991                                           "PCP value");
1992         if (vlan_m &&
1993             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1994             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1995                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1996             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1997             !(mlx5_flow_find_action
1998                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1999                 return rte_flow_error_set(error, EINVAL,
2000                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2001                                           "not full match mask on VLAN VID and "
2002                                           "there is no of_set_vlan_vid action, "
2003                                           "push VLAN action cannot figure out "
2004                                           "VID value");
2005         (void)attr;
2006         return 0;
2007 }
2008
2009 /**
2010  * Validate the set VLAN PCP.
2011  *
2012  * @param[in] action_flags
2013  *   Holds the actions detected until now.
2014  * @param[in] actions
2015  *   Pointer to the list of actions remaining in the flow rule.
2016  * @param[out] error
2017  *   Pointer to error structure.
2018  *
2019  * @return
2020  *   0 on success, a negative errno value otherwise and rte_errno is set.
2021  */
2022 static int
2023 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2024                                      const struct rte_flow_action actions[],
2025                                      struct rte_flow_error *error)
2026 {
2027         const struct rte_flow_action *action = actions;
2028         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2029
2030         if (conf->vlan_pcp > 7)
2031                 return rte_flow_error_set(error, EINVAL,
2032                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2033                                           "VLAN PCP value is too big");
2034         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2035                 return rte_flow_error_set(error, ENOTSUP,
2036                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2037                                           "set VLAN PCP action must follow "
2038                                           "the push VLAN action");
2039         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2040                 return rte_flow_error_set(error, ENOTSUP,
2041                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2042                                           "Multiple VLAN PCP modification are "
2043                                           "not supported");
2044         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2045                 return rte_flow_error_set(error, EINVAL,
2046                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2047                                           "wrong action order, port_id should "
2048                                           "be after set VLAN PCP");
2049         return 0;
2050 }
2051
2052 /**
2053  * Validate the set VLAN VID.
2054  *
2055  * @param[in] item_flags
2056  *   Holds the items detected in this rule.
2057  * @param[in] action_flags
2058  *   Holds the actions detected until now.
2059  * @param[in] actions
2060  *   Pointer to the list of actions remaining in the flow rule.
2061  * @param[out] error
2062  *   Pointer to error structure.
2063  *
2064  * @return
2065  *   0 on success, a negative errno value otherwise and rte_errno is set.
2066  */
2067 static int
2068 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2069                                      uint64_t action_flags,
2070                                      const struct rte_flow_action actions[],
2071                                      struct rte_flow_error *error)
2072 {
2073         const struct rte_flow_action *action = actions;
2074         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2075
2076         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2077                 return rte_flow_error_set(error, EINVAL,
2078                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2079                                           "VLAN VID value is too big");
2080         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2081             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2082                 return rte_flow_error_set(error, ENOTSUP,
2083                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2084                                           "set VLAN VID action must follow push"
2085                                           " VLAN action or match on VLAN item");
2086         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2087                 return rte_flow_error_set(error, ENOTSUP,
2088                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2089                                           "Multiple VLAN VID modifications are "
2090                                           "not supported");
2091         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2092                 return rte_flow_error_set(error, EINVAL,
2093                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2094                                           "wrong action order, port_id should "
2095                                           "be after set VLAN VID");
2096         return 0;
2097 }
2098
2099 /*
2100  * Validate the FLAG action.
2101  *
2102  * @param[in] dev
2103  *   Pointer to the rte_eth_dev structure.
2104  * @param[in] action_flags
2105  *   Holds the actions detected until now.
2106  * @param[in] attr
2107  *   Pointer to flow attributes
2108  * @param[out] error
2109  *   Pointer to error structure.
2110  *
2111  * @return
2112  *   0 on success, a negative errno value otherwise and rte_errno is set.
2113  */
2114 static int
2115 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2116                              uint64_t action_flags,
2117                              const struct rte_flow_attr *attr,
2118                              struct rte_flow_error *error)
2119 {
2120         struct mlx5_priv *priv = dev->data->dev_private;
2121         struct mlx5_dev_config *config = &priv->config;
2122         int ret;
2123
2124         /* Fall back if no extended metadata register support. */
2125         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2126                 return mlx5_flow_validate_action_flag(action_flags, attr,
2127                                                       error);
2128         /* Extensive metadata mode requires registers. */
2129         if (!mlx5_flow_ext_mreg_supported(dev))
2130                 return rte_flow_error_set(error, ENOTSUP,
2131                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2132                                           "no metadata registers "
2133                                           "to support flag action");
2134         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2135                 return rte_flow_error_set(error, ENOTSUP,
2136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2137                                           "extended metadata register"
2138                                           " isn't available");
2139         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2140         if (ret < 0)
2141                 return ret;
2142         MLX5_ASSERT(ret > 0);
2143         if (action_flags & MLX5_FLOW_ACTION_MARK)
2144                 return rte_flow_error_set(error, EINVAL,
2145                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2146                                           "can't mark and flag in same flow");
2147         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2148                 return rte_flow_error_set(error, EINVAL,
2149                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2150                                           "can't have 2 flag"
2151                                           " actions in same flow");
2152         return 0;
2153 }
2154
2155 /**
2156  * Validate MARK action.
2157  *
2158  * @param[in] dev
2159  *   Pointer to the rte_eth_dev structure.
2160  * @param[in] action
2161  *   Pointer to action.
2162  * @param[in] action_flags
2163  *   Holds the actions detected until now.
2164  * @param[in] attr
2165  *   Pointer to flow attributes
2166  * @param[out] error
2167  *   Pointer to error structure.
2168  *
2169  * @return
2170  *   0 on success, a negative errno value otherwise and rte_errno is set.
2171  */
2172 static int
2173 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2174                              const struct rte_flow_action *action,
2175                              uint64_t action_flags,
2176                              const struct rte_flow_attr *attr,
2177                              struct rte_flow_error *error)
2178 {
2179         struct mlx5_priv *priv = dev->data->dev_private;
2180         struct mlx5_dev_config *config = &priv->config;
2181         const struct rte_flow_action_mark *mark = action->conf;
2182         int ret;
2183
2184         /* Fall back if no extended metadata register support. */
2185         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2186                 return mlx5_flow_validate_action_mark(action, action_flags,
2187                                                       attr, error);
2188         /* Extensive metadata mode requires registers. */
2189         if (!mlx5_flow_ext_mreg_supported(dev))
2190                 return rte_flow_error_set(error, ENOTSUP,
2191                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2192                                           "no metadata registers "
2193                                           "to support mark action");
2194         if (!priv->sh->dv_mark_mask)
2195                 return rte_flow_error_set(error, ENOTSUP,
2196                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2197                                           "extended metadata register"
2198                                           " isn't available");
2199         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2200         if (ret < 0)
2201                 return ret;
2202         MLX5_ASSERT(ret > 0);
2203         if (!mark)
2204                 return rte_flow_error_set(error, EINVAL,
2205                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2206                                           "configuration cannot be null");
2207         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2208                 return rte_flow_error_set(error, EINVAL,
2209                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2210                                           &mark->id,
2211                                           "mark id exceeds the limit");
2212         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2215                                           "can't flag and mark in same flow");
2216         if (action_flags & MLX5_FLOW_ACTION_MARK)
2217                 return rte_flow_error_set(error, EINVAL,
2218                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2219                                           "can't have 2 mark actions in same"
2220                                           " flow");
2221         return 0;
2222 }
2223
2224 /**
2225  * Validate SET_META action.
2226  *
2227  * @param[in] dev
2228  *   Pointer to the rte_eth_dev structure.
2229  * @param[in] action
2230  *   Pointer to the action structure.
2231  * @param[in] action_flags
2232  *   Holds the actions detected until now.
2233  * @param[in] attr
2234  *   Pointer to flow attributes
2235  * @param[out] error
2236  *   Pointer to error structure.
2237  *
2238  * @return
2239  *   0 on success, a negative errno value otherwise and rte_errno is set.
2240  */
2241 static int
2242 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2243                                  const struct rte_flow_action *action,
2244                                  uint64_t action_flags __rte_unused,
2245                                  const struct rte_flow_attr *attr,
2246                                  struct rte_flow_error *error)
2247 {
2248         const struct rte_flow_action_set_meta *conf;
2249         uint32_t nic_mask = UINT32_MAX;
2250         int reg;
2251
2252         if (!mlx5_flow_ext_mreg_supported(dev))
2253                 return rte_flow_error_set(error, ENOTSUP,
2254                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2255                                           "extended metadata register"
2256                                           " isn't supported");
2257         reg = flow_dv_get_metadata_reg(dev, attr, error);
2258         if (reg < 0)
2259                 return reg;
2260         if (reg != REG_A && reg != REG_B) {
2261                 struct mlx5_priv *priv = dev->data->dev_private;
2262
2263                 nic_mask = priv->sh->dv_meta_mask;
2264         }
2265         if (!(action->conf))
2266                 return rte_flow_error_set(error, EINVAL,
2267                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2268                                           "configuration cannot be null");
2269         conf = (const struct rte_flow_action_set_meta *)action->conf;
2270         if (!conf->mask)
2271                 return rte_flow_error_set(error, EINVAL,
2272                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2273                                           "zero mask doesn't have any effect");
2274         if (conf->mask & ~nic_mask)
2275                 return rte_flow_error_set(error, EINVAL,
2276                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2277                                           "meta data must be within reg C0");
2278         return 0;
2279 }
2280
2281 /**
2282  * Validate SET_TAG action.
2283  *
2284  * @param[in] dev
2285  *   Pointer to the rte_eth_dev structure.
2286  * @param[in] action
2287  *   Pointer to the action structure.
2288  * @param[in] action_flags
2289  *   Holds the actions detected until now.
2290  * @param[in] attr
2291  *   Pointer to flow attributes
2292  * @param[out] error
2293  *   Pointer to error structure.
2294  *
2295  * @return
2296  *   0 on success, a negative errno value otherwise and rte_errno is set.
2297  */
2298 static int
2299 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2300                                 const struct rte_flow_action *action,
2301                                 uint64_t action_flags,
2302                                 const struct rte_flow_attr *attr,
2303                                 struct rte_flow_error *error)
2304 {
2305         const struct rte_flow_action_set_tag *conf;
2306         const uint64_t terminal_action_flags =
2307                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2308                 MLX5_FLOW_ACTION_RSS;
2309         int ret;
2310
2311         if (!mlx5_flow_ext_mreg_supported(dev))
2312                 return rte_flow_error_set(error, ENOTSUP,
2313                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2314                                           "extensive metadata register"
2315                                           " isn't supported");
2316         if (!(action->conf))
2317                 return rte_flow_error_set(error, EINVAL,
2318                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2319                                           "configuration cannot be null");
2320         conf = (const struct rte_flow_action_set_tag *)action->conf;
2321         if (!conf->mask)
2322                 return rte_flow_error_set(error, EINVAL,
2323                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2324                                           "zero mask doesn't have any effect");
2325         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2326         if (ret < 0)
2327                 return ret;
2328         if (!attr->transfer && attr->ingress &&
2329             (action_flags & terminal_action_flags))
2330                 return rte_flow_error_set(error, EINVAL,
2331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2332                                           "set_tag has no effect"
2333                                           " with terminal actions");
2334         return 0;
2335 }
2336
2337 /**
2338  * Validate count action.
2339  *
2340  * @param[in] dev
2341  *   Pointer to rte_eth_dev structure.
2342  * @param[out] error
2343  *   Pointer to error structure.
2344  *
2345  * @return
2346  *   0 on success, a negative errno value otherwise and rte_errno is set.
2347  */
2348 static int
2349 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2350                               struct rte_flow_error *error)
2351 {
2352         struct mlx5_priv *priv = dev->data->dev_private;
2353
2354         if (!priv->config.devx)
2355                 goto notsup_err;
2356 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2357         return 0;
2358 #endif
2359 notsup_err:
2360         return rte_flow_error_set
2361                       (error, ENOTSUP,
2362                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2363                        NULL,
2364                        "count action not supported");
2365 }
2366
2367 /**
2368  * Validate the L2 encap action.
2369  *
2370  * @param[in] dev
2371  *   Pointer to the rte_eth_dev structure.
2372  * @param[in] action_flags
2373  *   Holds the actions detected until now.
2374  * @param[in] action
2375  *   Pointer to the action structure.
2376  * @param[in] attr
2377  *   Pointer to flow attributes.
2378  * @param[out] error
2379  *   Pointer to error structure.
2380  *
2381  * @return
2382  *   0 on success, a negative errno value otherwise and rte_errno is set.
2383  */
2384 static int
2385 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2386                                  uint64_t action_flags,
2387                                  const struct rte_flow_action *action,
2388                                  const struct rte_flow_attr *attr,
2389                                  struct rte_flow_error *error)
2390 {
2391         const struct mlx5_priv *priv = dev->data->dev_private;
2392
2393         if (!(action->conf))
2394                 return rte_flow_error_set(error, EINVAL,
2395                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2396                                           "configuration cannot be null");
2397         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2398                 return rte_flow_error_set(error, EINVAL,
2399                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2400                                           "can only have a single encap action "
2401                                           "in a flow");
2402         if (!attr->transfer && priv->representor)
2403                 return rte_flow_error_set(error, ENOTSUP,
2404                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2405                                           "encap action for VF representor "
2406                                           "not supported on NIC table");
2407         return 0;
2408 }
2409
2410 /**
2411  * Validate a decap action.
2412  *
2413  * @param[in] dev
2414  *   Pointer to the rte_eth_dev structure.
2415  * @param[in] action_flags
2416  *   Holds the actions detected until now.
2417  * @param[in] attr
2418  *   Pointer to flow attributes
2419  * @param[out] error
2420  *   Pointer to error structure.
2421  *
2422  * @return
2423  *   0 on success, a negative errno value otherwise and rte_errno is set.
2424  */
2425 static int
2426 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2427                               uint64_t action_flags,
2428                               const struct rte_flow_attr *attr,
2429                               struct rte_flow_error *error)
2430 {
2431         const struct mlx5_priv *priv = dev->data->dev_private;
2432
2433         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2434             !priv->config.decap_en)
2435                 return rte_flow_error_set(error, ENOTSUP,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2437                                           "decap is not enabled");
2438         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2439                 return rte_flow_error_set(error, ENOTSUP,
2440                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2441                                           action_flags &
2442                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2443                                           "have a single decap action" : "decap "
2444                                           "after encap is not supported");
2445         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2446                 return rte_flow_error_set(error, EINVAL,
2447                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2448                                           "can't have decap action after"
2449                                           " modify action");
2450         if (attr->egress)
2451                 return rte_flow_error_set(error, ENOTSUP,
2452                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2453                                           NULL,
2454                                           "decap action not supported for "
2455                                           "egress");
2456         if (!attr->transfer && priv->representor)
2457                 return rte_flow_error_set(error, ENOTSUP,
2458                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2459                                           "decap action for VF representor "
2460                                           "not supported on NIC table");
2461         return 0;
2462 }
2463
2464 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2465
2466 /**
2467  * Validate the raw encap and decap actions.
2468  *
2469  * @param[in] dev
2470  *   Pointer to the rte_eth_dev structure.
2471  * @param[in] decap
2472  *   Pointer to the decap action.
2473  * @param[in] encap
2474  *   Pointer to the encap action.
2475  * @param[in] attr
2476  *   Pointer to flow attributes
2477  * @param[in/out] action_flags
2478  *   Holds the actions detected until now.
2479  * @param[out] actions_n
2480  *   pointer to the number of actions counter.
2481  * @param[out] error
2482  *   Pointer to error structure.
2483  *
2484  * @return
2485  *   0 on success, a negative errno value otherwise and rte_errno is set.
2486  */
2487 static int
2488 flow_dv_validate_action_raw_encap_decap
2489         (struct rte_eth_dev *dev,
2490          const struct rte_flow_action_raw_decap *decap,
2491          const struct rte_flow_action_raw_encap *encap,
2492          const struct rte_flow_attr *attr, uint64_t *action_flags,
2493          int *actions_n, struct rte_flow_error *error)
2494 {
2495         const struct mlx5_priv *priv = dev->data->dev_private;
2496         int ret;
2497
2498         if (encap && (!encap->size || !encap->data))
2499                 return rte_flow_error_set(error, EINVAL,
2500                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2501                                           "raw encap data cannot be empty");
2502         if (decap && encap) {
2503                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2504                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2505                         /* L3 encap. */
2506                         decap = NULL;
2507                 else if (encap->size <=
2508                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2509                            decap->size >
2510                            MLX5_ENCAPSULATION_DECISION_SIZE)
2511                         /* L3 decap. */
2512                         encap = NULL;
2513                 else if (encap->size >
2514                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2515                            decap->size >
2516                            MLX5_ENCAPSULATION_DECISION_SIZE)
2517                         /* 2 L2 actions: encap and decap. */
2518                         ;
2519                 else
2520                         return rte_flow_error_set(error,
2521                                 ENOTSUP,
2522                                 RTE_FLOW_ERROR_TYPE_ACTION,
2523                                 NULL, "unsupported too small "
2524                                 "raw decap and too small raw "
2525                                 "encap combination");
2526         }
2527         if (decap) {
2528                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2529                                                     error);
2530                 if (ret < 0)
2531                         return ret;
2532                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2533                 ++(*actions_n);
2534         }
2535         if (encap) {
2536                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2537                         return rte_flow_error_set(error, ENOTSUP,
2538                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2539                                                   NULL,
2540                                                   "small raw encap size");
2541                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2542                         return rte_flow_error_set(error, EINVAL,
2543                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2544                                                   NULL,
2545                                                   "more than one encap action");
2546                 if (!attr->transfer && priv->representor)
2547                         return rte_flow_error_set
2548                                         (error, ENOTSUP,
2549                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2550                                          "encap action for VF representor "
2551                                          "not supported on NIC table");
2552                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2553                 ++(*actions_n);
2554         }
2555         return 0;
2556 }
2557
2558 /**
2559  * Find existing encap/decap resource or create and register a new one.
2560  *
2561  * @param[in, out] dev
2562  *   Pointer to rte_eth_dev structure.
2563  * @param[in, out] resource
2564  *   Pointer to encap/decap resource.
2565  * @parm[in, out] dev_flow
2566  *   Pointer to the dev_flow.
2567  * @param[out] error
2568  *   pointer to error structure.
2569  *
2570  * @return
2571  *   0 on success otherwise -errno and errno is set.
2572  */
2573 static int
2574 flow_dv_encap_decap_resource_register
2575                         (struct rte_eth_dev *dev,
2576                          struct mlx5_flow_dv_encap_decap_resource *resource,
2577                          struct mlx5_flow *dev_flow,
2578                          struct rte_flow_error *error)
2579 {
2580         struct mlx5_priv *priv = dev->data->dev_private;
2581         struct mlx5_dev_ctx_shared *sh = priv->sh;
2582         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2583         struct mlx5dv_dr_domain *domain;
2584         uint32_t idx = 0;
2585         int ret;
2586
2587         resource->flags = dev_flow->dv.group ? 0 : 1;
2588         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2589                 domain = sh->fdb_domain;
2590         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2591                 domain = sh->rx_domain;
2592         else
2593                 domain = sh->tx_domain;
2594         /* Lookup a matching resource from cache. */
2595         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2596                       cache_resource, next) {
2597                 if (resource->reformat_type == cache_resource->reformat_type &&
2598                     resource->ft_type == cache_resource->ft_type &&
2599                     resource->flags == cache_resource->flags &&
2600                     resource->size == cache_resource->size &&
2601                     !memcmp((const void *)resource->buf,
2602                             (const void *)cache_resource->buf,
2603                             resource->size)) {
2604                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2605                                 (void *)cache_resource,
2606                                 rte_atomic32_read(&cache_resource->refcnt));
2607                         rte_atomic32_inc(&cache_resource->refcnt);
2608                         dev_flow->handle->dvh.rix_encap_decap = idx;
2609                         dev_flow->dv.encap_decap = cache_resource;
2610                         return 0;
2611                 }
2612         }
2613         /* Register new encap/decap resource. */
2614         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2615                                        &dev_flow->handle->dvh.rix_encap_decap);
2616         if (!cache_resource)
2617                 return rte_flow_error_set(error, ENOMEM,
2618                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2619                                           "cannot allocate resource memory");
2620         *cache_resource = *resource;
2621         ret = mlx5_flow_os_create_flow_action_packet_reformat
2622                                         (sh->ctx, domain, cache_resource,
2623                                          &cache_resource->action);
2624         if (ret) {
2625                 mlx5_free(cache_resource);
2626                 return rte_flow_error_set(error, ENOMEM,
2627                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2628                                           NULL, "cannot create action");
2629         }
2630         rte_atomic32_init(&cache_resource->refcnt);
2631         rte_atomic32_inc(&cache_resource->refcnt);
2632         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2633                      dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2634                      next);
2635         dev_flow->dv.encap_decap = cache_resource;
2636         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2637                 (void *)cache_resource,
2638                 rte_atomic32_read(&cache_resource->refcnt));
2639         return 0;
2640 }
2641
2642 /**
2643  * Find existing table jump resource or create and register a new one.
2644  *
2645  * @param[in, out] dev
2646  *   Pointer to rte_eth_dev structure.
2647  * @param[in, out] tbl
2648  *   Pointer to flow table resource.
2649  * @parm[in, out] dev_flow
2650  *   Pointer to the dev_flow.
2651  * @param[out] error
2652  *   pointer to error structure.
2653  *
2654  * @return
2655  *   0 on success otherwise -errno and errno is set.
2656  */
2657 static int
2658 flow_dv_jump_tbl_resource_register
2659                         (struct rte_eth_dev *dev __rte_unused,
2660                          struct mlx5_flow_tbl_resource *tbl,
2661                          struct mlx5_flow *dev_flow,
2662                          struct rte_flow_error *error)
2663 {
2664         struct mlx5_flow_tbl_data_entry *tbl_data =
2665                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2666         int cnt, ret;
2667
2668         MLX5_ASSERT(tbl);
2669         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2670         if (!cnt) {
2671                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2672                                 (tbl->obj, &tbl_data->jump.action);
2673                 if (ret)
2674                         return rte_flow_error_set(error, ENOMEM,
2675                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2676                                         NULL, "cannot create jump action");
2677                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2678                         (void *)&tbl_data->jump, cnt);
2679         } else {
2680                 /* old jump should not make the table ref++. */
2681                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2682                 MLX5_ASSERT(tbl_data->jump.action);
2683                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2684                         (void *)&tbl_data->jump, cnt);
2685         }
2686         rte_atomic32_inc(&tbl_data->jump.refcnt);
2687         dev_flow->handle->rix_jump = tbl_data->idx;
2688         dev_flow->dv.jump = &tbl_data->jump;
2689         return 0;
2690 }
2691
2692 /**
2693  * Find existing default miss resource or create and register a new one.
2694  *
2695  * @param[in, out] dev
2696  *   Pointer to rte_eth_dev structure.
2697  * @param[out] error
2698  *   pointer to error structure.
2699  *
2700  * @return
2701  *   0 on success otherwise -errno and errno is set.
2702  */
2703 static int
2704 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2705                 struct rte_flow_error *error)
2706 {
2707         struct mlx5_priv *priv = dev->data->dev_private;
2708         struct mlx5_dev_ctx_shared *sh = priv->sh;
2709         struct mlx5_flow_default_miss_resource *cache_resource =
2710                         &sh->default_miss;
2711         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2712
2713         if (!cnt) {
2714                 MLX5_ASSERT(cache_resource->action);
2715                 cache_resource->action =
2716                 mlx5_glue->dr_create_flow_action_default_miss();
2717                 if (!cache_resource->action)
2718                         return rte_flow_error_set(error, ENOMEM,
2719                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2720                                         "cannot create default miss action");
2721                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2722                                 (void *)cache_resource->action, cnt);
2723         }
2724         rte_atomic32_inc(&cache_resource->refcnt);
2725         return 0;
2726 }
2727
2728 /**
2729  * Find existing table port ID resource or create and register a new one.
2730  *
2731  * @param[in, out] dev
2732  *   Pointer to rte_eth_dev structure.
2733  * @param[in, out] resource
2734  *   Pointer to port ID action resource.
2735  * @parm[in, out] dev_flow
2736  *   Pointer to the dev_flow.
2737  * @param[out] error
2738  *   pointer to error structure.
2739  *
2740  * @return
2741  *   0 on success otherwise -errno and errno is set.
2742  */
2743 static int
2744 flow_dv_port_id_action_resource_register
2745                         (struct rte_eth_dev *dev,
2746                          struct mlx5_flow_dv_port_id_action_resource *resource,
2747                          struct mlx5_flow *dev_flow,
2748                          struct rte_flow_error *error)
2749 {
2750         struct mlx5_priv *priv = dev->data->dev_private;
2751         struct mlx5_dev_ctx_shared *sh = priv->sh;
2752         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2753         uint32_t idx = 0;
2754         int ret;
2755
2756         /* Lookup a matching resource from cache. */
2757         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2758                       idx, cache_resource, next) {
2759                 if (resource->port_id == cache_resource->port_id) {
2760                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2761                                 "refcnt %d++",
2762                                 (void *)cache_resource,
2763                                 rte_atomic32_read(&cache_resource->refcnt));
2764                         rte_atomic32_inc(&cache_resource->refcnt);
2765                         dev_flow->handle->rix_port_id_action = idx;
2766                         dev_flow->dv.port_id_action = cache_resource;
2767                         return 0;
2768                 }
2769         }
2770         /* Register new port id action resource. */
2771         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2772                                        &dev_flow->handle->rix_port_id_action);
2773         if (!cache_resource)
2774                 return rte_flow_error_set(error, ENOMEM,
2775                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2776                                           "cannot allocate resource memory");
2777         *cache_resource = *resource;
2778         ret = mlx5_flow_os_create_flow_action_dest_port
2779                                 (priv->sh->fdb_domain, resource->port_id,
2780                                  &cache_resource->action);
2781         if (ret) {
2782                 mlx5_free(cache_resource);
2783                 return rte_flow_error_set(error, ENOMEM,
2784                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2785                                           NULL, "cannot create action");
2786         }
2787         rte_atomic32_init(&cache_resource->refcnt);
2788         rte_atomic32_inc(&cache_resource->refcnt);
2789         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2790                      dev_flow->handle->rix_port_id_action, cache_resource,
2791                      next);
2792         dev_flow->dv.port_id_action = cache_resource;
2793         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2794                 (void *)cache_resource,
2795                 rte_atomic32_read(&cache_resource->refcnt));
2796         return 0;
2797 }
2798
2799 /**
2800  * Find existing push vlan resource or create and register a new one.
2801  *
2802  * @param [in, out] dev
2803  *   Pointer to rte_eth_dev structure.
2804  * @param[in, out] resource
2805  *   Pointer to port ID action resource.
2806  * @parm[in, out] dev_flow
2807  *   Pointer to the dev_flow.
2808  * @param[out] error
2809  *   pointer to error structure.
2810  *
2811  * @return
2812  *   0 on success otherwise -errno and errno is set.
2813  */
2814 static int
2815 flow_dv_push_vlan_action_resource_register
2816                        (struct rte_eth_dev *dev,
2817                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2818                         struct mlx5_flow *dev_flow,
2819                         struct rte_flow_error *error)
2820 {
2821         struct mlx5_priv *priv = dev->data->dev_private;
2822         struct mlx5_dev_ctx_shared *sh = priv->sh;
2823         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2824         struct mlx5dv_dr_domain *domain;
2825         uint32_t idx = 0;
2826         int ret;
2827
2828         /* Lookup a matching resource from cache. */
2829         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2830                       sh->push_vlan_action_list, idx, cache_resource, next) {
2831                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2832                     resource->ft_type == cache_resource->ft_type) {
2833                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2834                                 "refcnt %d++",
2835                                 (void *)cache_resource,
2836                                 rte_atomic32_read(&cache_resource->refcnt));
2837                         rte_atomic32_inc(&cache_resource->refcnt);
2838                         dev_flow->handle->dvh.rix_push_vlan = idx;
2839                         dev_flow->dv.push_vlan_res = cache_resource;
2840                         return 0;
2841                 }
2842         }
2843         /* Register new push_vlan action resource. */
2844         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2845                                        &dev_flow->handle->dvh.rix_push_vlan);
2846         if (!cache_resource)
2847                 return rte_flow_error_set(error, ENOMEM,
2848                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2849                                           "cannot allocate resource memory");
2850         *cache_resource = *resource;
2851         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2852                 domain = sh->fdb_domain;
2853         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2854                 domain = sh->rx_domain;
2855         else
2856                 domain = sh->tx_domain;
2857         ret = mlx5_flow_os_create_flow_action_push_vlan
2858                                         (domain, resource->vlan_tag,
2859                                          &cache_resource->action);
2860         if (ret) {
2861                 mlx5_free(cache_resource);
2862                 return rte_flow_error_set(error, ENOMEM,
2863                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2864                                           NULL, "cannot create action");
2865         }
2866         rte_atomic32_init(&cache_resource->refcnt);
2867         rte_atomic32_inc(&cache_resource->refcnt);
2868         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2869                      &sh->push_vlan_action_list,
2870                      dev_flow->handle->dvh.rix_push_vlan,
2871                      cache_resource, next);
2872         dev_flow->dv.push_vlan_res = cache_resource;
2873         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2874                 (void *)cache_resource,
2875                 rte_atomic32_read(&cache_resource->refcnt));
2876         return 0;
2877 }
2878 /**
2879  * Get the size of specific rte_flow_item_type
2880  *
2881  * @param[in] item_type
2882  *   Tested rte_flow_item_type.
2883  *
2884  * @return
2885  *   sizeof struct item_type, 0 if void or irrelevant.
2886  */
2887 static size_t
2888 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2889 {
2890         size_t retval;
2891
2892         switch (item_type) {
2893         case RTE_FLOW_ITEM_TYPE_ETH:
2894                 retval = sizeof(struct rte_flow_item_eth);
2895                 break;
2896         case RTE_FLOW_ITEM_TYPE_VLAN:
2897                 retval = sizeof(struct rte_flow_item_vlan);
2898                 break;
2899         case RTE_FLOW_ITEM_TYPE_IPV4:
2900                 retval = sizeof(struct rte_flow_item_ipv4);
2901                 break;
2902         case RTE_FLOW_ITEM_TYPE_IPV6:
2903                 retval = sizeof(struct rte_flow_item_ipv6);
2904                 break;
2905         case RTE_FLOW_ITEM_TYPE_UDP:
2906                 retval = sizeof(struct rte_flow_item_udp);
2907                 break;
2908         case RTE_FLOW_ITEM_TYPE_TCP:
2909                 retval = sizeof(struct rte_flow_item_tcp);
2910                 break;
2911         case RTE_FLOW_ITEM_TYPE_VXLAN:
2912                 retval = sizeof(struct rte_flow_item_vxlan);
2913                 break;
2914         case RTE_FLOW_ITEM_TYPE_GRE:
2915                 retval = sizeof(struct rte_flow_item_gre);
2916                 break;
2917         case RTE_FLOW_ITEM_TYPE_NVGRE:
2918                 retval = sizeof(struct rte_flow_item_nvgre);
2919                 break;
2920         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2921                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2922                 break;
2923         case RTE_FLOW_ITEM_TYPE_MPLS:
2924                 retval = sizeof(struct rte_flow_item_mpls);
2925                 break;
2926         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2927         default:
2928                 retval = 0;
2929                 break;
2930         }
2931         return retval;
2932 }
2933
2934 #define MLX5_ENCAP_IPV4_VERSION         0x40
2935 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2936 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2937 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2938 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2939 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2940 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2941
2942 /**
2943  * Convert the encap action data from list of rte_flow_item to raw buffer
2944  *
2945  * @param[in] items
2946  *   Pointer to rte_flow_item objects list.
2947  * @param[out] buf
2948  *   Pointer to the output buffer.
2949  * @param[out] size
2950  *   Pointer to the output buffer size.
2951  * @param[out] error
2952  *   Pointer to the error structure.
2953  *
2954  * @return
2955  *   0 on success, a negative errno value otherwise and rte_errno is set.
2956  */
2957 static int
2958 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2959                            size_t *size, struct rte_flow_error *error)
2960 {
2961         struct rte_ether_hdr *eth = NULL;
2962         struct rte_vlan_hdr *vlan = NULL;
2963         struct rte_ipv4_hdr *ipv4 = NULL;
2964         struct rte_ipv6_hdr *ipv6 = NULL;
2965         struct rte_udp_hdr *udp = NULL;
2966         struct rte_vxlan_hdr *vxlan = NULL;
2967         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2968         struct rte_gre_hdr *gre = NULL;
2969         size_t len;
2970         size_t temp_size = 0;
2971
2972         if (!items)
2973                 return rte_flow_error_set(error, EINVAL,
2974                                           RTE_FLOW_ERROR_TYPE_ACTION,
2975                                           NULL, "invalid empty data");
2976         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2977                 len = flow_dv_get_item_len(items->type);
2978                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2979                         return rte_flow_error_set(error, EINVAL,
2980                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2981                                                   (void *)items->type,
2982                                                   "items total size is too big"
2983                                                   " for encap action");
2984                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2985                 switch (items->type) {
2986                 case RTE_FLOW_ITEM_TYPE_ETH:
2987                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2988                         break;
2989                 case RTE_FLOW_ITEM_TYPE_VLAN:
2990                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2991                         if (!eth)
2992                                 return rte_flow_error_set(error, EINVAL,
2993                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2994                                                 (void *)items->type,
2995                                                 "eth header not found");
2996                         if (!eth->ether_type)
2997                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2998                         break;
2999                 case RTE_FLOW_ITEM_TYPE_IPV4:
3000                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3001                         if (!vlan && !eth)
3002                                 return rte_flow_error_set(error, EINVAL,
3003                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3004                                                 (void *)items->type,
3005                                                 "neither eth nor vlan"
3006                                                 " header found");
3007                         if (vlan && !vlan->eth_proto)
3008                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3009                         else if (eth && !eth->ether_type)
3010                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3011                         if (!ipv4->version_ihl)
3012                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3013                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3014                         if (!ipv4->time_to_live)
3015                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3016                         break;
3017                 case RTE_FLOW_ITEM_TYPE_IPV6:
3018                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3019                         if (!vlan && !eth)
3020                                 return rte_flow_error_set(error, EINVAL,
3021                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3022                                                 (void *)items->type,
3023                                                 "neither eth nor vlan"
3024                                                 " header found");
3025                         if (vlan && !vlan->eth_proto)
3026                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3027                         else if (eth && !eth->ether_type)
3028                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3029                         if (!ipv6->vtc_flow)
3030                                 ipv6->vtc_flow =
3031                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3032                         if (!ipv6->hop_limits)
3033                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3034                         break;
3035                 case RTE_FLOW_ITEM_TYPE_UDP:
3036                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3037                         if (!ipv4 && !ipv6)
3038                                 return rte_flow_error_set(error, EINVAL,
3039                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3040                                                 (void *)items->type,
3041                                                 "ip header not found");
3042                         if (ipv4 && !ipv4->next_proto_id)
3043                                 ipv4->next_proto_id = IPPROTO_UDP;
3044                         else if (ipv6 && !ipv6->proto)
3045                                 ipv6->proto = IPPROTO_UDP;
3046                         break;
3047                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3048                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3049                         if (!udp)
3050                                 return rte_flow_error_set(error, EINVAL,
3051                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3052                                                 (void *)items->type,
3053                                                 "udp header not found");
3054                         if (!udp->dst_port)
3055                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3056                         if (!vxlan->vx_flags)
3057                                 vxlan->vx_flags =
3058                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3059                         break;
3060                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3061                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3062                         if (!udp)
3063                                 return rte_flow_error_set(error, EINVAL,
3064                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3065                                                 (void *)items->type,
3066                                                 "udp header not found");
3067                         if (!vxlan_gpe->proto)
3068                                 return rte_flow_error_set(error, EINVAL,
3069                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3070                                                 (void *)items->type,
3071                                                 "next protocol not found");
3072                         if (!udp->dst_port)
3073                                 udp->dst_port =
3074                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3075                         if (!vxlan_gpe->vx_flags)
3076                                 vxlan_gpe->vx_flags =
3077                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3078                         break;
3079                 case RTE_FLOW_ITEM_TYPE_GRE:
3080                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3081                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3082                         if (!gre->proto)
3083                                 return rte_flow_error_set(error, EINVAL,
3084                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3085                                                 (void *)items->type,
3086                                                 "next protocol not found");
3087                         if (!ipv4 && !ipv6)
3088                                 return rte_flow_error_set(error, EINVAL,
3089                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3090                                                 (void *)items->type,
3091                                                 "ip header not found");
3092                         if (ipv4 && !ipv4->next_proto_id)
3093                                 ipv4->next_proto_id = IPPROTO_GRE;
3094                         else if (ipv6 && !ipv6->proto)
3095                                 ipv6->proto = IPPROTO_GRE;
3096                         break;
3097                 case RTE_FLOW_ITEM_TYPE_VOID:
3098                         break;
3099                 default:
3100                         return rte_flow_error_set(error, EINVAL,
3101                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3102                                                   (void *)items->type,
3103                                                   "unsupported item type");
3104                         break;
3105                 }
3106                 temp_size += len;
3107         }
3108         *size = temp_size;
3109         return 0;
3110 }
3111
3112 static int
3113 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3114 {
3115         struct rte_ether_hdr *eth = NULL;
3116         struct rte_vlan_hdr *vlan = NULL;
3117         struct rte_ipv6_hdr *ipv6 = NULL;
3118         struct rte_udp_hdr *udp = NULL;
3119         char *next_hdr;
3120         uint16_t proto;
3121
3122         eth = (struct rte_ether_hdr *)data;
3123         next_hdr = (char *)(eth + 1);
3124         proto = RTE_BE16(eth->ether_type);
3125
3126         /* VLAN skipping */
3127         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3128                 vlan = (struct rte_vlan_hdr *)next_hdr;
3129                 proto = RTE_BE16(vlan->eth_proto);
3130                 next_hdr += sizeof(struct rte_vlan_hdr);
3131         }
3132
3133         /* HW calculates IPv4 csum. no need to proceed */
3134         if (proto == RTE_ETHER_TYPE_IPV4)
3135                 return 0;
3136
3137         /* non IPv4/IPv6 header. not supported */
3138         if (proto != RTE_ETHER_TYPE_IPV6) {
3139                 return rte_flow_error_set(error, ENOTSUP,
3140                                           RTE_FLOW_ERROR_TYPE_ACTION,
3141                                           NULL, "Cannot offload non IPv4/IPv6");
3142         }
3143
3144         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3145
3146         /* ignore non UDP */
3147         if (ipv6->proto != IPPROTO_UDP)
3148                 return 0;
3149
3150         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3151         udp->dgram_cksum = 0;
3152
3153         return 0;
3154 }
3155
3156 /**
3157  * Convert L2 encap action to DV specification.
3158  *
3159  * @param[in] dev
3160  *   Pointer to rte_eth_dev structure.
3161  * @param[in] action
3162  *   Pointer to action structure.
3163  * @param[in, out] dev_flow
3164  *   Pointer to the mlx5_flow.
3165  * @param[in] transfer
3166  *   Mark if the flow is E-Switch flow.
3167  * @param[out] error
3168  *   Pointer to the error structure.
3169  *
3170  * @return
3171  *   0 on success, a negative errno value otherwise and rte_errno is set.
3172  */
3173 static int
3174 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3175                                const struct rte_flow_action *action,
3176                                struct mlx5_flow *dev_flow,
3177                                uint8_t transfer,
3178                                struct rte_flow_error *error)
3179 {
3180         const struct rte_flow_item *encap_data;
3181         const struct rte_flow_action_raw_encap *raw_encap_data;
3182         struct mlx5_flow_dv_encap_decap_resource res = {
3183                 .reformat_type =
3184                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3185                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3186                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3187         };
3188
3189         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3190                 raw_encap_data =
3191                         (const struct rte_flow_action_raw_encap *)action->conf;
3192                 res.size = raw_encap_data->size;
3193                 memcpy(res.buf, raw_encap_data->data, res.size);
3194         } else {
3195                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3196                         encap_data =
3197                                 ((const struct rte_flow_action_vxlan_encap *)
3198                                                 action->conf)->definition;
3199                 else
3200                         encap_data =
3201                                 ((const struct rte_flow_action_nvgre_encap *)
3202                                                 action->conf)->definition;
3203                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3204                                                &res.size, error))
3205                         return -rte_errno;
3206         }
3207         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3208                 return -rte_errno;
3209         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3210                 return rte_flow_error_set(error, EINVAL,
3211                                           RTE_FLOW_ERROR_TYPE_ACTION,
3212                                           NULL, "can't create L2 encap action");
3213         return 0;
3214 }
3215
3216 /**
3217  * Convert L2 decap action to DV specification.
3218  *
3219  * @param[in] dev
3220  *   Pointer to rte_eth_dev structure.
3221  * @param[in, out] dev_flow
3222  *   Pointer to the mlx5_flow.
3223  * @param[in] transfer
3224  *   Mark if the flow is E-Switch flow.
3225  * @param[out] error
3226  *   Pointer to the error structure.
3227  *
3228  * @return
3229  *   0 on success, a negative errno value otherwise and rte_errno is set.
3230  */
3231 static int
3232 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3233                                struct mlx5_flow *dev_flow,
3234                                uint8_t transfer,
3235                                struct rte_flow_error *error)
3236 {
3237         struct mlx5_flow_dv_encap_decap_resource res = {
3238                 .size = 0,
3239                 .reformat_type =
3240                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3241                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3242                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3243         };
3244
3245         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "can't create L2 decap action");
3249         return 0;
3250 }
3251
3252 /**
3253  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3254  *
3255  * @param[in] dev
3256  *   Pointer to rte_eth_dev structure.
3257  * @param[in] action
3258  *   Pointer to action structure.
3259  * @param[in, out] dev_flow
3260  *   Pointer to the mlx5_flow.
3261  * @param[in] attr
3262  *   Pointer to the flow attributes.
3263  * @param[out] error
3264  *   Pointer to the error structure.
3265  *
3266  * @return
3267  *   0 on success, a negative errno value otherwise and rte_errno is set.
3268  */
3269 static int
3270 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3271                                 const struct rte_flow_action *action,
3272                                 struct mlx5_flow *dev_flow,
3273                                 const struct rte_flow_attr *attr,
3274                                 struct rte_flow_error *error)
3275 {
3276         const struct rte_flow_action_raw_encap *encap_data;
3277         struct mlx5_flow_dv_encap_decap_resource res;
3278
3279         memset(&res, 0, sizeof(res));
3280         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3281         res.size = encap_data->size;
3282         memcpy(res.buf, encap_data->data, res.size);
3283         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3284                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3285                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3286         if (attr->transfer)
3287                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3288         else
3289                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3290                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3291         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3292                 return rte_flow_error_set(error, EINVAL,
3293                                           RTE_FLOW_ERROR_TYPE_ACTION,
3294                                           NULL, "can't create encap action");
3295         return 0;
3296 }
3297
3298 /**
3299  * Create action push VLAN.
3300  *
3301  * @param[in] dev
3302  *   Pointer to rte_eth_dev structure.
3303  * @param[in] attr
3304  *   Pointer to the flow attributes.
3305  * @param[in] vlan
3306  *   Pointer to the vlan to push to the Ethernet header.
3307  * @param[in, out] dev_flow
3308  *   Pointer to the mlx5_flow.
3309  * @param[out] error
3310  *   Pointer to the error structure.
3311  *
3312  * @return
3313  *   0 on success, a negative errno value otherwise and rte_errno is set.
3314  */
3315 static int
3316 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3317                                 const struct rte_flow_attr *attr,
3318                                 const struct rte_vlan_hdr *vlan,
3319                                 struct mlx5_flow *dev_flow,
3320                                 struct rte_flow_error *error)
3321 {
3322         struct mlx5_flow_dv_push_vlan_action_resource res;
3323
3324         memset(&res, 0, sizeof(res));
3325         res.vlan_tag =
3326                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3327                                  vlan->vlan_tci);
3328         if (attr->transfer)
3329                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3330         else
3331                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3332                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3333         return flow_dv_push_vlan_action_resource_register
3334                                             (dev, &res, dev_flow, error);
3335 }
3336
3337 /**
3338  * Validate the modify-header actions.
3339  *
3340  * @param[in] action_flags
3341  *   Holds the actions detected until now.
3342  * @param[in] action
3343  *   Pointer to the modify action.
3344  * @param[out] error
3345  *   Pointer to error structure.
3346  *
3347  * @return
3348  *   0 on success, a negative errno value otherwise and rte_errno is set.
3349  */
3350 static int
3351 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3352                                    const struct rte_flow_action *action,
3353                                    struct rte_flow_error *error)
3354 {
3355         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3356                 return rte_flow_error_set(error, EINVAL,
3357                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3358                                           NULL, "action configuration not set");
3359         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3360                 return rte_flow_error_set(error, EINVAL,
3361                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3362                                           "can't have encap action before"
3363                                           " modify action");
3364         return 0;
3365 }
3366
3367 /**
3368  * Validate the modify-header MAC address actions.
3369  *
3370  * @param[in] action_flags
3371  *   Holds the actions detected until now.
3372  * @param[in] action
3373  *   Pointer to the modify action.
3374  * @param[in] item_flags
3375  *   Holds the items detected.
3376  * @param[out] error
3377  *   Pointer to error structure.
3378  *
3379  * @return
3380  *   0 on success, a negative errno value otherwise and rte_errno is set.
3381  */
3382 static int
3383 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3384                                    const struct rte_flow_action *action,
3385                                    const uint64_t item_flags,
3386                                    struct rte_flow_error *error)
3387 {
3388         int ret = 0;
3389
3390         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3391         if (!ret) {
3392                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3393                         return rte_flow_error_set(error, EINVAL,
3394                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3395                                                   NULL,
3396                                                   "no L2 item in pattern");
3397         }
3398         return ret;
3399 }
3400
3401 /**
3402  * Validate the modify-header IPv4 address actions.
3403  *
3404  * @param[in] action_flags
3405  *   Holds the actions detected until now.
3406  * @param[in] action
3407  *   Pointer to the modify action.
3408  * @param[in] item_flags
3409  *   Holds the items detected.
3410  * @param[out] error
3411  *   Pointer to error structure.
3412  *
3413  * @return
3414  *   0 on success, a negative errno value otherwise and rte_errno is set.
3415  */
3416 static int
3417 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3418                                     const struct rte_flow_action *action,
3419                                     const uint64_t item_flags,
3420                                     struct rte_flow_error *error)
3421 {
3422         int ret = 0;
3423         uint64_t layer;
3424
3425         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3426         if (!ret) {
3427                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3428                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3429                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3430                 if (!(item_flags & layer))
3431                         return rte_flow_error_set(error, EINVAL,
3432                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3433                                                   NULL,
3434                                                   "no ipv4 item in pattern");
3435         }
3436         return ret;
3437 }
3438
3439 /**
3440  * Validate the modify-header IPv6 address actions.
3441  *
3442  * @param[in] action_flags
3443  *   Holds the actions detected until now.
3444  * @param[in] action
3445  *   Pointer to the modify action.
3446  * @param[in] item_flags
3447  *   Holds the items detected.
3448  * @param[out] error
3449  *   Pointer to error structure.
3450  *
3451  * @return
3452  *   0 on success, a negative errno value otherwise and rte_errno is set.
3453  */
3454 static int
3455 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3456                                     const struct rte_flow_action *action,
3457                                     const uint64_t item_flags,
3458                                     struct rte_flow_error *error)
3459 {
3460         int ret = 0;
3461         uint64_t layer;
3462
3463         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3464         if (!ret) {
3465                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3466                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3467                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3468                 if (!(item_flags & layer))
3469                         return rte_flow_error_set(error, EINVAL,
3470                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3471                                                   NULL,
3472                                                   "no ipv6 item in pattern");
3473         }
3474         return ret;
3475 }
3476
3477 /**
3478  * Validate the modify-header TP actions.
3479  *
3480  * @param[in] action_flags
3481  *   Holds the actions detected until now.
3482  * @param[in] action
3483  *   Pointer to the modify action.
3484  * @param[in] item_flags
3485  *   Holds the items detected.
3486  * @param[out] error
3487  *   Pointer to error structure.
3488  *
3489  * @return
3490  *   0 on success, a negative errno value otherwise and rte_errno is set.
3491  */
3492 static int
3493 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3494                                   const struct rte_flow_action *action,
3495                                   const uint64_t item_flags,
3496                                   struct rte_flow_error *error)
3497 {
3498         int ret = 0;
3499         uint64_t layer;
3500
3501         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3502         if (!ret) {
3503                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3504                                  MLX5_FLOW_LAYER_INNER_L4 :
3505                                  MLX5_FLOW_LAYER_OUTER_L4;
3506                 if (!(item_flags & layer))
3507                         return rte_flow_error_set(error, EINVAL,
3508                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3509                                                   NULL, "no transport layer "
3510                                                   "in pattern");
3511         }
3512         return ret;
3513 }
3514
3515 /**
3516  * Validate the modify-header actions of increment/decrement
3517  * TCP Sequence-number.
3518  *
3519  * @param[in] action_flags
3520  *   Holds the actions detected until now.
3521  * @param[in] action
3522  *   Pointer to the modify action.
3523  * @param[in] item_flags
3524  *   Holds the items detected.
3525  * @param[out] error
3526  *   Pointer to error structure.
3527  *
3528  * @return
3529  *   0 on success, a negative errno value otherwise and rte_errno is set.
3530  */
3531 static int
3532 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3533                                        const struct rte_flow_action *action,
3534                                        const uint64_t item_flags,
3535                                        struct rte_flow_error *error)
3536 {
3537         int ret = 0;
3538         uint64_t layer;
3539
3540         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3541         if (!ret) {
3542                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3543                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3544                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3545                 if (!(item_flags & layer))
3546                         return rte_flow_error_set(error, EINVAL,
3547                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3548                                                   NULL, "no TCP item in"
3549                                                   " pattern");
3550                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3551                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3552                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3553                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3554                         return rte_flow_error_set(error, EINVAL,
3555                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3556                                                   NULL,
3557                                                   "cannot decrease and increase"
3558                                                   " TCP sequence number"
3559                                                   " at the same time");
3560         }
3561         return ret;
3562 }
3563
3564 /**
3565  * Validate the modify-header actions of increment/decrement
3566  * TCP Acknowledgment number.
3567  *
3568  * @param[in] action_flags
3569  *   Holds the actions detected until now.
3570  * @param[in] action
3571  *   Pointer to the modify action.
3572  * @param[in] item_flags
3573  *   Holds the items detected.
3574  * @param[out] error
3575  *   Pointer to error structure.
3576  *
3577  * @return
3578  *   0 on success, a negative errno value otherwise and rte_errno is set.
3579  */
3580 static int
3581 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3582                                        const struct rte_flow_action *action,
3583                                        const uint64_t item_flags,
3584                                        struct rte_flow_error *error)
3585 {
3586         int ret = 0;
3587         uint64_t layer;
3588
3589         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3590         if (!ret) {
3591                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3592                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3593                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3594                 if (!(item_flags & layer))
3595                         return rte_flow_error_set(error, EINVAL,
3596                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3597                                                   NULL, "no TCP item in"
3598                                                   " pattern");
3599                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3600                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3601                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3602                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3603                         return rte_flow_error_set(error, EINVAL,
3604                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3605                                                   NULL,
3606                                                   "cannot decrease and increase"
3607                                                   " TCP acknowledgment number"
3608                                                   " at the same time");
3609         }
3610         return ret;
3611 }
3612
3613 /**
3614  * Validate the modify-header TTL actions.
3615  *
3616  * @param[in] action_flags
3617  *   Holds the actions detected until now.
3618  * @param[in] action
3619  *   Pointer to the modify action.
3620  * @param[in] item_flags
3621  *   Holds the items detected.
3622  * @param[out] error
3623  *   Pointer to error structure.
3624  *
3625  * @return
3626  *   0 on success, a negative errno value otherwise and rte_errno is set.
3627  */
3628 static int
3629 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3630                                    const struct rte_flow_action *action,
3631                                    const uint64_t item_flags,
3632                                    struct rte_flow_error *error)
3633 {
3634         int ret = 0;
3635         uint64_t layer;
3636
3637         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3638         if (!ret) {
3639                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3640                                  MLX5_FLOW_LAYER_INNER_L3 :
3641                                  MLX5_FLOW_LAYER_OUTER_L3;
3642                 if (!(item_flags & layer))
3643                         return rte_flow_error_set(error, EINVAL,
3644                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3645                                                   NULL,
3646                                                   "no IP protocol in pattern");
3647         }
3648         return ret;
3649 }
3650
3651 /**
3652  * Validate jump action.
3653  *
3654  * @param[in] action
3655  *   Pointer to the jump action.
3656  * @param[in] action_flags
3657  *   Holds the actions detected until now.
3658  * @param[in] attributes
3659  *   Pointer to flow attributes
3660  * @param[in] external
3661  *   Action belongs to flow rule created by request external to PMD.
3662  * @param[out] error
3663  *   Pointer to error structure.
3664  *
3665  * @return
3666  *   0 on success, a negative errno value otherwise and rte_errno is set.
3667  */
3668 static int
3669 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3670                              uint64_t action_flags,
3671                              const struct rte_flow_attr *attributes,
3672                              bool external, struct rte_flow_error *error)
3673 {
3674         uint32_t target_group, table;
3675         int ret = 0;
3676
3677         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3678                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3679                 return rte_flow_error_set(error, EINVAL,
3680                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3681                                           "can't have 2 fate actions in"
3682                                           " same flow");
3683         if (action_flags & MLX5_FLOW_ACTION_METER)
3684                 return rte_flow_error_set(error, ENOTSUP,
3685                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3686                                           "jump with meter not support");
3687         if (!action->conf)
3688                 return rte_flow_error_set(error, EINVAL,
3689                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3690                                           NULL, "action configuration not set");
3691         target_group =
3692                 ((const struct rte_flow_action_jump *)action->conf)->group;
3693         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3694                                        true, &table, error);
3695         if (ret)
3696                 return ret;
3697         if (attributes->group == target_group)
3698                 return rte_flow_error_set(error, EINVAL,
3699                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3700                                           "target group must be other than"
3701                                           " the current flow group");
3702         return 0;
3703 }
3704
3705 /*
3706  * Validate the port_id action.
3707  *
3708  * @param[in] dev
3709  *   Pointer to rte_eth_dev structure.
3710  * @param[in] action_flags
3711  *   Bit-fields that holds the actions detected until now.
3712  * @param[in] action
3713  *   Port_id RTE action structure.
3714  * @param[in] attr
3715  *   Attributes of flow that includes this action.
3716  * @param[out] error
3717  *   Pointer to error structure.
3718  *
3719  * @return
3720  *   0 on success, a negative errno value otherwise and rte_errno is set.
3721  */
3722 static int
3723 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3724                                 uint64_t action_flags,
3725                                 const struct rte_flow_action *action,
3726                                 const struct rte_flow_attr *attr,
3727                                 struct rte_flow_error *error)
3728 {
3729         const struct rte_flow_action_port_id *port_id;
3730         struct mlx5_priv *act_priv;
3731         struct mlx5_priv *dev_priv;
3732         uint16_t port;
3733
3734         if (!attr->transfer)
3735                 return rte_flow_error_set(error, ENOTSUP,
3736                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3737                                           NULL,
3738                                           "port id action is valid in transfer"
3739                                           " mode only");
3740         if (!action || !action->conf)
3741                 return rte_flow_error_set(error, ENOTSUP,
3742                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3743                                           NULL,
3744                                           "port id action parameters must be"
3745                                           " specified");
3746         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3747                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3748                 return rte_flow_error_set(error, EINVAL,
3749                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3750                                           "can have only one fate actions in"
3751                                           " a flow");
3752         dev_priv = mlx5_dev_to_eswitch_info(dev);
3753         if (!dev_priv)
3754                 return rte_flow_error_set(error, rte_errno,
3755                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3756                                           NULL,
3757                                           "failed to obtain E-Switch info");
3758         port_id = action->conf;
3759         port = port_id->original ? dev->data->port_id : port_id->id;
3760         act_priv = mlx5_port_to_eswitch_info(port, false);
3761         if (!act_priv)
3762                 return rte_flow_error_set
3763                                 (error, rte_errno,
3764                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3765                                  "failed to obtain E-Switch port id for port");
3766         if (act_priv->domain_id != dev_priv->domain_id)
3767                 return rte_flow_error_set
3768                                 (error, EINVAL,
3769                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3770                                  "port does not belong to"
3771                                  " E-Switch being configured");
3772         return 0;
3773 }
3774
3775 /**
3776  * Get the maximum number of modify header actions.
3777  *
3778  * @param dev
3779  *   Pointer to rte_eth_dev structure.
3780  * @param flags
3781  *   Flags bits to check if root level.
3782  *
3783  * @return
3784  *   Max number of modify header actions device can support.
3785  */
3786 static inline unsigned int
3787 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3788                               uint64_t flags)
3789 {
3790         /*
3791          * There's no way to directly query the max capacity from FW.
3792          * The maximal value on root table should be assumed to be supported.
3793          */
3794         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3795                 return MLX5_MAX_MODIFY_NUM;
3796         else
3797                 return MLX5_ROOT_TBL_MODIFY_NUM;
3798 }
3799
3800 /**
3801  * Validate the meter action.
3802  *
3803  * @param[in] dev
3804  *   Pointer to rte_eth_dev structure.
3805  * @param[in] action_flags
3806  *   Bit-fields that holds the actions detected until now.
3807  * @param[in] action
3808  *   Pointer to the meter action.
3809  * @param[in] attr
3810  *   Attributes of flow that includes this action.
3811  * @param[out] error
3812  *   Pointer to error structure.
3813  *
3814  * @return
3815  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3816  */
3817 static int
3818 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3819                                 uint64_t action_flags,
3820                                 const struct rte_flow_action *action,
3821                                 const struct rte_flow_attr *attr,
3822                                 struct rte_flow_error *error)
3823 {
3824         struct mlx5_priv *priv = dev->data->dev_private;
3825         const struct rte_flow_action_meter *am = action->conf;
3826         struct mlx5_flow_meter *fm;
3827
3828         if (!am)
3829                 return rte_flow_error_set(error, EINVAL,
3830                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3831                                           "meter action conf is NULL");
3832
3833         if (action_flags & MLX5_FLOW_ACTION_METER)
3834                 return rte_flow_error_set(error, ENOTSUP,
3835                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3836                                           "meter chaining not support");
3837         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3838                 return rte_flow_error_set(error, ENOTSUP,
3839                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3840                                           "meter with jump not support");
3841         if (!priv->mtr_en)
3842                 return rte_flow_error_set(error, ENOTSUP,
3843                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3844                                           NULL,
3845                                           "meter action not supported");
3846         fm = mlx5_flow_meter_find(priv, am->mtr_id);
3847         if (!fm)
3848                 return rte_flow_error_set(error, EINVAL,
3849                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3850                                           "Meter not found");
3851         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3852               (!fm->ingress && !attr->ingress && attr->egress) ||
3853               (!fm->egress && !attr->egress && attr->ingress))))
3854                 return rte_flow_error_set(error, EINVAL,
3855                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3856                                           "Flow attributes are either invalid "
3857                                           "or have a conflict with current "
3858                                           "meter attributes");
3859         return 0;
3860 }
3861
3862 /**
3863  * Validate the age action.
3864  *
3865  * @param[in] action_flags
3866  *   Holds the actions detected until now.
3867  * @param[in] action
3868  *   Pointer to the age action.
3869  * @param[in] dev
3870  *   Pointer to the Ethernet device structure.
3871  * @param[out] error
3872  *   Pointer to error structure.
3873  *
3874  * @return
3875  *   0 on success, a negative errno value otherwise and rte_errno is set.
3876  */
3877 static int
3878 flow_dv_validate_action_age(uint64_t action_flags,
3879                             const struct rte_flow_action *action,
3880                             struct rte_eth_dev *dev,
3881                             struct rte_flow_error *error)
3882 {
3883         struct mlx5_priv *priv = dev->data->dev_private;
3884         const struct rte_flow_action_age *age = action->conf;
3885
3886         if (!priv->config.devx || priv->counter_fallback)
3887                 return rte_flow_error_set(error, ENOTSUP,
3888                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3889                                           NULL,
3890                                           "age action not supported");
3891         if (!(action->conf))
3892                 return rte_flow_error_set(error, EINVAL,
3893                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3894                                           "configuration cannot be null");
3895         if (age->timeout >= UINT16_MAX / 2 / 10)
3896                 return rte_flow_error_set(error, ENOTSUP,
3897                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3898                                           "Max age time: 3275 seconds");
3899         if (action_flags & MLX5_FLOW_ACTION_AGE)
3900                 return rte_flow_error_set(error, EINVAL,
3901                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3902                                           "Duplicate age ctions set");
3903         return 0;
3904 }
3905
3906 /**
3907  * Validate the modify-header IPv4 DSCP actions.
3908  *
3909  * @param[in] action_flags
3910  *   Holds the actions detected until now.
3911  * @param[in] action
3912  *   Pointer to the modify action.
3913  * @param[in] item_flags
3914  *   Holds the items detected.
3915  * @param[out] error
3916  *   Pointer to error structure.
3917  *
3918  * @return
3919  *   0 on success, a negative errno value otherwise and rte_errno is set.
3920  */
3921 static int
3922 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3923                                          const struct rte_flow_action *action,
3924                                          const uint64_t item_flags,
3925                                          struct rte_flow_error *error)
3926 {
3927         int ret = 0;
3928
3929         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3930         if (!ret) {
3931                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3932                         return rte_flow_error_set(error, EINVAL,
3933                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3934                                                   NULL,
3935                                                   "no ipv4 item in pattern");
3936         }
3937         return ret;
3938 }
3939
3940 /**
3941  * Validate the modify-header IPv6 DSCP actions.
3942  *
3943  * @param[in] action_flags
3944  *   Holds the actions detected until now.
3945  * @param[in] action
3946  *   Pointer to the modify action.
3947  * @param[in] item_flags
3948  *   Holds the items detected.
3949  * @param[out] error
3950  *   Pointer to error structure.
3951  *
3952  * @return
3953  *   0 on success, a negative errno value otherwise and rte_errno is set.
3954  */
3955 static int
3956 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3957                                          const struct rte_flow_action *action,
3958                                          const uint64_t item_flags,
3959                                          struct rte_flow_error *error)
3960 {
3961         int ret = 0;
3962
3963         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3964         if (!ret) {
3965                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3966                         return rte_flow_error_set(error, EINVAL,
3967                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3968                                                   NULL,
3969                                                   "no ipv6 item in pattern");
3970         }
3971         return ret;
3972 }
3973
3974 /**
3975  * Find existing modify-header resource or create and register a new one.
3976  *
3977  * @param dev[in, out]
3978  *   Pointer to rte_eth_dev structure.
3979  * @param[in, out] resource
3980  *   Pointer to modify-header resource.
3981  * @parm[in, out] dev_flow
3982  *   Pointer to the dev_flow.
3983  * @param[out] error
3984  *   pointer to error structure.
3985  *
3986  * @return
3987  *   0 on success otherwise -errno and errno is set.
3988  */
3989 static int
3990 flow_dv_modify_hdr_resource_register
3991                         (struct rte_eth_dev *dev,
3992                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3993                          struct mlx5_flow *dev_flow,
3994                          struct rte_flow_error *error)
3995 {
3996         struct mlx5_priv *priv = dev->data->dev_private;
3997         struct mlx5_dev_ctx_shared *sh = priv->sh;
3998         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3999         struct mlx5dv_dr_domain *ns;
4000         uint32_t actions_len;
4001         int ret;
4002
4003         resource->flags = dev_flow->dv.group ? 0 :
4004                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4005         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4006                                     resource->flags))
4007                 return rte_flow_error_set(error, EOVERFLOW,
4008                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4009                                           "too many modify header items");
4010         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4011                 ns = sh->fdb_domain;
4012         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4013                 ns = sh->tx_domain;
4014         else
4015                 ns = sh->rx_domain;
4016         /* Lookup a matching resource from cache. */
4017         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4018         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
4019                 if (resource->ft_type == cache_resource->ft_type &&
4020                     resource->actions_num == cache_resource->actions_num &&
4021                     resource->flags == cache_resource->flags &&
4022                     !memcmp((const void *)resource->actions,
4023                             (const void *)cache_resource->actions,
4024                             actions_len)) {
4025                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4026                                 (void *)cache_resource,
4027                                 rte_atomic32_read(&cache_resource->refcnt));
4028                         rte_atomic32_inc(&cache_resource->refcnt);
4029                         dev_flow->handle->dvh.modify_hdr = cache_resource;
4030                         return 0;
4031                 }
4032         }
4033         /* Register new modify-header resource. */
4034         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4035                                     sizeof(*cache_resource) + actions_len, 0,
4036                                     SOCKET_ID_ANY);
4037         if (!cache_resource)
4038                 return rte_flow_error_set(error, ENOMEM,
4039                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4040                                           "cannot allocate resource memory");
4041         *cache_resource = *resource;
4042         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4043         ret = mlx5_flow_os_create_flow_action_modify_header
4044                                         (sh->ctx, ns, cache_resource,
4045                                          actions_len, &cache_resource->action);
4046         if (ret) {
4047                 mlx5_free(cache_resource);
4048                 return rte_flow_error_set(error, ENOMEM,
4049                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4050                                           NULL, "cannot create action");
4051         }
4052         rte_atomic32_init(&cache_resource->refcnt);
4053         rte_atomic32_inc(&cache_resource->refcnt);
4054         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4055         dev_flow->handle->dvh.modify_hdr = cache_resource;
4056         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4057                 (void *)cache_resource,
4058                 rte_atomic32_read(&cache_resource->refcnt));
4059         return 0;
4060 }
4061
4062 /**
4063  * Get DV flow counter by index.
4064  *
4065  * @param[in] dev
4066  *   Pointer to the Ethernet device structure.
4067  * @param[in] idx
4068  *   mlx5 flow counter index in the container.
4069  * @param[out] ppool
4070  *   mlx5 flow counter pool in the container,
4071  *
4072  * @return
4073  *   Pointer to the counter, NULL otherwise.
4074  */
4075 static struct mlx5_flow_counter *
4076 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4077                            uint32_t idx,
4078                            struct mlx5_flow_counter_pool **ppool)
4079 {
4080         struct mlx5_priv *priv = dev->data->dev_private;
4081         struct mlx5_pools_container *cont;
4082         struct mlx5_flow_counter_pool *pool;
4083         uint32_t batch = 0, age = 0;
4084
4085         idx--;
4086         age = MLX_CNT_IS_AGE(idx);
4087         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4088         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4089                 idx -= MLX5_CNT_BATCH_OFFSET;
4090                 batch = 1;
4091         }
4092         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4093         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4094         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4095         MLX5_ASSERT(pool);
4096         if (ppool)
4097                 *ppool = pool;
4098         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4099 }
4100
4101 /**
4102  * Check the devx counter belongs to the pool.
4103  *
4104  * @param[in] pool
4105  *   Pointer to the counter pool.
4106  * @param[in] id
4107  *   The counter devx ID.
4108  *
4109  * @return
4110  *   True if counter belongs to the pool, false otherwise.
4111  */
4112 static bool
4113 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4114 {
4115         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4116                    MLX5_COUNTERS_PER_POOL;
4117
4118         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4119                 return true;
4120         return false;
4121 }
4122
4123 /**
4124  * Get a pool by devx counter ID.
4125  *
4126  * @param[in] cont
4127  *   Pointer to the counter container.
4128  * @param[in] id
4129  *   The counter devx ID.
4130  *
4131  * @return
4132  *   The counter pool pointer if exists, NULL otherwise,
4133  */
4134 static struct mlx5_flow_counter_pool *
4135 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4136 {
4137         uint32_t i;
4138
4139         /* Check last used pool. */
4140         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4141             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4142                 return cont->pools[cont->last_pool_idx];
4143         /* ID out of range means no suitable pool in the container. */
4144         if (id > cont->max_id || id < cont->min_id)
4145                 return NULL;
4146         /*
4147          * Find the pool from the end of the container, since mostly counter
4148          * ID is sequence increasing, and the last pool should be the needed
4149          * one.
4150          */
4151         i = rte_atomic16_read(&cont->n_valid);
4152         while (i--) {
4153                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4154
4155                 if (flow_dv_is_counter_in_pool(pool, id))
4156                         return pool;
4157         }
4158         return NULL;
4159 }
4160
4161 /**
4162  * Allocate a new memory for the counter values wrapped by all the needed
4163  * management.
4164  *
4165  * @param[in] dev
4166  *   Pointer to the Ethernet device structure.
4167  * @param[in] raws_n
4168  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4169  *
4170  * @return
4171  *   The new memory management pointer on success, otherwise NULL and rte_errno
4172  *   is set.
4173  */
4174 static struct mlx5_counter_stats_mem_mng *
4175 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4176 {
4177         struct mlx5_priv *priv = dev->data->dev_private;
4178         struct mlx5_dev_ctx_shared *sh = priv->sh;
4179         struct mlx5_devx_mkey_attr mkey_attr;
4180         struct mlx5_counter_stats_mem_mng *mem_mng;
4181         volatile struct flow_counter_stats *raw_data;
4182         int size = (sizeof(struct flow_counter_stats) *
4183                         MLX5_COUNTERS_PER_POOL +
4184                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4185                         sizeof(struct mlx5_counter_stats_mem_mng);
4186         size_t pgsize = rte_mem_page_size();
4187         if (pgsize == (size_t)-1) {
4188                 DRV_LOG(ERR, "Failed to get mem page size");
4189                 rte_errno = ENOMEM;
4190                 return NULL;
4191         }
4192         uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
4193                                   SOCKET_ID_ANY);
4194         int i;
4195
4196         if (!mem) {
4197                 rte_errno = ENOMEM;
4198                 return NULL;
4199         }
4200         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4201         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4202         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4203                                                  IBV_ACCESS_LOCAL_WRITE);
4204         if (!mem_mng->umem) {
4205                 rte_errno = errno;
4206                 mlx5_free(mem);
4207                 return NULL;
4208         }
4209         mkey_attr.addr = (uintptr_t)mem;
4210         mkey_attr.size = size;
4211         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4212         mkey_attr.pd = sh->pdn;
4213         mkey_attr.log_entity_size = 0;
4214         mkey_attr.pg_access = 0;
4215         mkey_attr.klm_array = NULL;
4216         mkey_attr.klm_num = 0;
4217         if (priv->config.hca_attr.relaxed_ordering_write &&
4218                 priv->config.hca_attr.relaxed_ordering_read  &&
4219                 !haswell_broadwell_cpu)
4220                 mkey_attr.relaxed_ordering = 1;
4221         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4222         if (!mem_mng->dm) {
4223                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4224                 rte_errno = errno;
4225                 mlx5_free(mem);
4226                 return NULL;
4227         }
4228         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4229         raw_data = (volatile struct flow_counter_stats *)mem;
4230         for (i = 0; i < raws_n; ++i) {
4231                 mem_mng->raws[i].mem_mng = mem_mng;
4232                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4233         }
4234         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4235         return mem_mng;
4236 }
4237
4238 /**
4239  * Resize a counter container.
4240  *
4241  * @param[in] dev
4242  *   Pointer to the Ethernet device structure.
4243  * @param[in] batch
4244  *   Whether the pool is for counter that was allocated by batch command.
4245  * @param[in] age
4246  *   Whether the pool is for Aging counter.
4247  *
4248  * @return
4249  *   0 on success, otherwise negative errno value and rte_errno is set.
4250  */
4251 static int
4252 flow_dv_container_resize(struct rte_eth_dev *dev,
4253                                 uint32_t batch, uint32_t age)
4254 {
4255         struct mlx5_priv *priv = dev->data->dev_private;
4256         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4257                                                                age);
4258         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4259         void *old_pools = cont->pools;
4260         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4261         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4262         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4263
4264         if (!pools) {
4265                 rte_errno = ENOMEM;
4266                 return -ENOMEM;
4267         }
4268         if (old_pools)
4269                 memcpy(pools, old_pools, cont->n *
4270                                        sizeof(struct mlx5_flow_counter_pool *));
4271         /*
4272          * Fallback mode query the counter directly, no background query
4273          * resources are needed.
4274          */
4275         if (!priv->counter_fallback) {
4276                 int i;
4277
4278                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4279                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4280                 if (!mem_mng) {
4281                         mlx5_free(pools);
4282                         return -ENOMEM;
4283                 }
4284                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4285                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4286                                          mem_mng->raws +
4287                                          MLX5_CNT_CONTAINER_RESIZE +
4288                                          i, next);
4289         }
4290         rte_spinlock_lock(&cont->resize_sl);
4291         cont->n = resize;
4292         cont->mem_mng = mem_mng;
4293         cont->pools = pools;
4294         rte_spinlock_unlock(&cont->resize_sl);
4295         if (old_pools)
4296                 mlx5_free(old_pools);
4297         return 0;
4298 }
4299
4300 /**
4301  * Query a devx flow counter.
4302  *
4303  * @param[in] dev
4304  *   Pointer to the Ethernet device structure.
4305  * @param[in] cnt
4306  *   Index to the flow counter.
4307  * @param[out] pkts
4308  *   The statistics value of packets.
4309  * @param[out] bytes
4310  *   The statistics value of bytes.
4311  *
4312  * @return
4313  *   0 on success, otherwise a negative errno value and rte_errno is set.
4314  */
4315 static inline int
4316 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4317                      uint64_t *bytes)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_flow_counter_pool *pool = NULL;
4321         struct mlx5_flow_counter *cnt;
4322         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4323         int offset;
4324
4325         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4326         MLX5_ASSERT(pool);
4327         if (counter < MLX5_CNT_BATCH_OFFSET) {
4328                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4329                 if (priv->counter_fallback)
4330                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4331                                         0, pkts, bytes, 0, NULL, NULL, 0);
4332         }
4333
4334         rte_spinlock_lock(&pool->sl);
4335         /*
4336          * The single counters allocation may allocate smaller ID than the
4337          * current allocated in parallel to the host reading.
4338          * In this case the new counter values must be reported as 0.
4339          */
4340         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4341                 *pkts = 0;
4342                 *bytes = 0;
4343         } else {
4344                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4345                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4346                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4347         }
4348         rte_spinlock_unlock(&pool->sl);
4349         return 0;
4350 }
4351
4352 /**
4353  * Create and initialize a new counter pool.
4354  *
4355  * @param[in] dev
4356  *   Pointer to the Ethernet device structure.
4357  * @param[out] dcs
4358  *   The devX counter handle.
4359  * @param[in] batch
4360  *   Whether the pool is for counter that was allocated by batch command.
4361  * @param[in] age
4362  *   Whether the pool is for counter that was allocated for aging.
4363  * @param[in/out] cont_cur
4364  *   Pointer to the container pointer, it will be update in pool resize.
4365  *
4366  * @return
4367  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4368  */
4369 static struct mlx5_flow_counter_pool *
4370 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4371                     uint32_t batch, uint32_t age)
4372 {
4373         struct mlx5_priv *priv = dev->data->dev_private;
4374         struct mlx5_flow_counter_pool *pool;
4375         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4376                                                                age);
4377         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4378         uint32_t size = sizeof(*pool);
4379
4380         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4381                 return NULL;
4382         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4383         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4384         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4385         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4386         if (!pool) {
4387                 rte_errno = ENOMEM;
4388                 return NULL;
4389         }
4390         pool->min_dcs = dcs;
4391         if (!priv->counter_fallback)
4392                 pool->raw = cont->mem_mng->raws + n_valid %
4393                                                       MLX5_CNT_CONTAINER_RESIZE;
4394         pool->raw_hw = NULL;
4395         pool->type = 0;
4396         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4397         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4398         pool->query_gen = 0;
4399         rte_spinlock_init(&pool->sl);
4400         TAILQ_INIT(&pool->counters[0]);
4401         TAILQ_INIT(&pool->counters[1]);
4402         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4403         pool->index = n_valid;
4404         cont->pools[n_valid] = pool;
4405         if (!batch) {
4406                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4407
4408                 if (base < cont->min_id)
4409                         cont->min_id = base;
4410                 if (base > cont->max_id)
4411                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4412                 cont->last_pool_idx = pool->index;
4413         }
4414         /* Pool initialization must be updated before host thread access. */
4415         rte_cio_wmb();
4416         rte_atomic16_add(&cont->n_valid, 1);
4417         return pool;
4418 }
4419
4420 /**
4421  * Update the minimum dcs-id for aged or no-aged counter pool.
4422  *
4423  * @param[in] dev
4424  *   Pointer to the Ethernet device structure.
4425  * @param[in] pool
4426  *   Current counter pool.
4427  * @param[in] batch
4428  *   Whether the pool is for counter that was allocated by batch command.
4429  * @param[in] age
4430  *   Whether the counter is for aging.
4431  */
4432 static void
4433 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4434                         struct mlx5_flow_counter_pool *pool,
4435                         uint32_t batch, uint32_t age)
4436 {
4437         struct mlx5_priv *priv = dev->data->dev_private;
4438         struct mlx5_flow_counter_pool *other;
4439         struct mlx5_pools_container *cont;
4440
4441         cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4442         other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4443         if (!other)
4444                 return;
4445         if (pool->min_dcs->id < other->min_dcs->id) {
4446                 rte_atomic64_set(&other->a64_dcs,
4447                         rte_atomic64_read(&pool->a64_dcs));
4448         } else {
4449                 rte_atomic64_set(&pool->a64_dcs,
4450                         rte_atomic64_read(&other->a64_dcs));
4451         }
4452 }
4453 /**
4454  * Prepare a new counter and/or a new counter pool.
4455  *
4456  * @param[in] dev
4457  *   Pointer to the Ethernet device structure.
4458  * @param[out] cnt_free
4459  *   Where to put the pointer of a new counter.
4460  * @param[in] batch
4461  *   Whether the pool is for counter that was allocated by batch command.
4462  * @param[in] age
4463  *   Whether the pool is for counter that was allocated for aging.
4464  *
4465  * @return
4466  *   The counter pool pointer and @p cnt_free is set on success,
4467  *   NULL otherwise and rte_errno is set.
4468  */
4469 static struct mlx5_flow_counter_pool *
4470 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4471                              struct mlx5_flow_counter **cnt_free,
4472                              uint32_t batch, uint32_t age)
4473 {
4474         struct mlx5_priv *priv = dev->data->dev_private;
4475         struct mlx5_pools_container *cont;
4476         struct mlx5_flow_counter_pool *pool;
4477         struct mlx5_counters tmp_tq;
4478         struct mlx5_devx_obj *dcs = NULL;
4479         struct mlx5_flow_counter *cnt;
4480         uint32_t i;
4481
4482         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4483         if (!batch) {
4484                 /* bulk_bitmap must be 0 for single counter allocation. */
4485                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4486                 if (!dcs)
4487                         return NULL;
4488                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4489                 if (!pool) {
4490                         pool = flow_dv_pool_create(dev, dcs, batch, age);
4491                         if (!pool) {
4492                                 mlx5_devx_cmd_destroy(dcs);
4493                                 return NULL;
4494                         }
4495                 } else if (dcs->id < pool->min_dcs->id) {
4496                         rte_atomic64_set(&pool->a64_dcs,
4497                                          (int64_t)(uintptr_t)dcs);
4498                 }
4499                 flow_dv_counter_update_min_dcs(dev,
4500                                                 pool, batch, age);
4501                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4502                 cnt = MLX5_POOL_GET_CNT(pool, i);
4503                 cnt->pool = pool;
4504                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4505                 *cnt_free = cnt;
4506                 return pool;
4507         }
4508         /* bulk_bitmap is in 128 counters units. */
4509         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4510                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4511         if (!dcs) {
4512                 rte_errno = ENODATA;
4513                 return NULL;
4514         }
4515         pool = flow_dv_pool_create(dev, dcs, batch, age);
4516         if (!pool) {
4517                 mlx5_devx_cmd_destroy(dcs);
4518                 return NULL;
4519         }
4520         TAILQ_INIT(&tmp_tq);
4521         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4522                 cnt = MLX5_POOL_GET_CNT(pool, i);
4523                 cnt->pool = pool;
4524                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4525         }
4526         rte_spinlock_lock(&cont->csl);
4527         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4528         rte_spinlock_unlock(&cont->csl);
4529         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4530         (*cnt_free)->pool = pool;
4531         return pool;
4532 }
4533
4534 /**
4535  * Search for existed shared counter.
4536  *
4537  * @param[in] dev
4538  *   Pointer to the Ethernet device structure.
4539  * @param[in] id
4540  *   The shared counter ID to search.
4541  * @param[out] ppool
4542  *   mlx5 flow counter pool in the container,
4543  *
4544  * @return
4545  *   NULL if not existed, otherwise pointer to the shared extend counter.
4546  */
4547 static struct mlx5_flow_counter_ext *
4548 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4549                               struct mlx5_flow_counter_pool **ppool)
4550 {
4551         struct mlx5_priv *priv = dev->data->dev_private;
4552         union mlx5_l3t_data data;
4553         uint32_t cnt_idx;
4554
4555         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4556                 return NULL;
4557         cnt_idx = data.dword;
4558         /*
4559          * Shared counters don't have age info. The counter extend is after
4560          * the counter datat structure.
4561          */
4562         return (struct mlx5_flow_counter_ext *)
4563                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4564 }
4565
4566 /**
4567  * Allocate a flow counter.
4568  *
4569  * @param[in] dev
4570  *   Pointer to the Ethernet device structure.
4571  * @param[in] shared
4572  *   Indicate if this counter is shared with other flows.
4573  * @param[in] id
4574  *   Counter identifier.
4575  * @param[in] group
4576  *   Counter flow group.
4577  * @param[in] age
4578  *   Whether the counter was allocated for aging.
4579  *
4580  * @return
4581  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4582  */
4583 static uint32_t
4584 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4585                       uint16_t group, uint32_t age)
4586 {
4587         struct mlx5_priv *priv = dev->data->dev_private;
4588         struct mlx5_flow_counter_pool *pool = NULL;
4589         struct mlx5_flow_counter *cnt_free = NULL;
4590         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4591         /*
4592          * Currently group 0 flow counter cannot be assigned to a flow if it is
4593          * not the first one in the batch counter allocation, so it is better
4594          * to allocate counters one by one for these flows in a separate
4595          * container.
4596          * A counter can be shared between different groups so need to take
4597          * shared counters from the single container.
4598          */
4599         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4600         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4601                                                                age);
4602         uint32_t cnt_idx;
4603
4604         if (!priv->config.devx) {
4605                 rte_errno = ENOTSUP;
4606                 return 0;
4607         }
4608         if (shared) {
4609                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4610                 if (cnt_ext) {
4611                         if (cnt_ext->ref_cnt + 1 == 0) {
4612                                 rte_errno = E2BIG;
4613                                 return 0;
4614                         }
4615                         cnt_ext->ref_cnt++;
4616                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4617                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4618                                   + 1;
4619                         return cnt_idx;
4620                 }
4621         }
4622         /* Get free counters from container. */
4623         rte_spinlock_lock(&cont->csl);
4624         cnt_free = TAILQ_FIRST(&cont->counters);
4625         if (cnt_free)
4626                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4627         rte_spinlock_unlock(&cont->csl);
4628         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4629                                                        batch, age))
4630                 goto err;
4631         pool = cnt_free->pool;
4632         if (!batch)
4633                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4634         /* Create a DV counter action only in the first time usage. */
4635         if (!cnt_free->action) {
4636                 uint16_t offset;
4637                 struct mlx5_devx_obj *dcs;
4638                 int ret;
4639
4640                 if (batch) {
4641                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4642                         dcs = pool->min_dcs;
4643                 } else {
4644                         offset = 0;
4645                         dcs = cnt_ext->dcs;
4646                 }
4647                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4648                                                             &cnt_free->action);
4649                 if (ret) {
4650                         rte_errno = errno;
4651                         goto err;
4652                 }
4653         }
4654         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4655                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4656         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4657         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4658         /* Update the counter reset values. */
4659         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4660                                  &cnt_free->bytes))
4661                 goto err;
4662         if (cnt_ext) {
4663                 cnt_ext->shared = shared;
4664                 cnt_ext->ref_cnt = 1;
4665                 cnt_ext->id = id;
4666                 if (shared) {
4667                         union mlx5_l3t_data data;
4668
4669                         data.dword = cnt_idx;
4670                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4671                                 return 0;
4672                 }
4673         }
4674         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4675                 /* Start the asynchronous batch query by the host thread. */
4676                 mlx5_set_query_alarm(priv->sh);
4677         return cnt_idx;
4678 err:
4679         if (cnt_free) {
4680                 cnt_free->pool = pool;
4681                 rte_spinlock_lock(&cont->csl);
4682                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4683                 rte_spinlock_unlock(&cont->csl);
4684         }
4685         return 0;
4686 }
4687
4688 /**
4689  * Get age param from counter index.
4690  *
4691  * @param[in] dev
4692  *   Pointer to the Ethernet device structure.
4693  * @param[in] counter
4694  *   Index to the counter handler.
4695  *
4696  * @return
4697  *   The aging parameter specified for the counter index.
4698  */
4699 static struct mlx5_age_param*
4700 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4701                                 uint32_t counter)
4702 {
4703         struct mlx5_flow_counter *cnt;
4704         struct mlx5_flow_counter_pool *pool = NULL;
4705
4706         flow_dv_counter_get_by_idx(dev, counter, &pool);
4707         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4708         cnt = MLX5_POOL_GET_CNT(pool, counter);
4709         return MLX5_CNT_TO_AGE(cnt);
4710 }
4711
4712 /**
4713  * Remove a flow counter from aged counter list.
4714  *
4715  * @param[in] dev
4716  *   Pointer to the Ethernet device structure.
4717  * @param[in] counter
4718  *   Index to the counter handler.
4719  * @param[in] cnt
4720  *   Pointer to the counter handler.
4721  */
4722 static void
4723 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4724                                 uint32_t counter, struct mlx5_flow_counter *cnt)
4725 {
4726         struct mlx5_age_info *age_info;
4727         struct mlx5_age_param *age_param;
4728         struct mlx5_priv *priv = dev->data->dev_private;
4729
4730         age_info = GET_PORT_AGE_INFO(priv);
4731         age_param = flow_dv_counter_idx_get_age(dev, counter);
4732         if (rte_atomic16_cmpset((volatile uint16_t *)
4733                         &age_param->state,
4734                         AGE_CANDIDATE, AGE_FREE)
4735                         != AGE_CANDIDATE) {
4736                 /**
4737                  * We need the lock even it is age timeout,
4738                  * since counter may still in process.
4739                  */
4740                 rte_spinlock_lock(&age_info->aged_sl);
4741                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4742                 rte_spinlock_unlock(&age_info->aged_sl);
4743         }
4744         rte_atomic16_set(&age_param->state, AGE_FREE);
4745 }
4746 /**
4747  * Release a flow counter.
4748  *
4749  * @param[in] dev
4750  *   Pointer to the Ethernet device structure.
4751  * @param[in] counter
4752  *   Index to the counter handler.
4753  */
4754 static void
4755 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4756 {
4757         struct mlx5_priv *priv = dev->data->dev_private;
4758         struct mlx5_flow_counter_pool *pool = NULL;
4759         struct mlx5_flow_counter *cnt;
4760         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4761
4762         if (!counter)
4763                 return;
4764         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4765         MLX5_ASSERT(pool);
4766         if (counter < MLX5_CNT_BATCH_OFFSET) {
4767                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4768                 if (cnt_ext) {
4769                         if (--cnt_ext->ref_cnt)
4770                                 return;
4771                         if (cnt_ext->shared)
4772                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4773                                                      cnt_ext->id);
4774                 }
4775         }
4776         if (IS_AGE_POOL(pool))
4777                 flow_dv_counter_remove_from_age(dev, counter, cnt);
4778         cnt->pool = pool;
4779         /*
4780          * Put the counter back to list to be updated in none fallback mode.
4781          * Currently, we are using two list alternately, while one is in query,
4782          * add the freed counter to the other list based on the pool query_gen
4783          * value. After query finishes, add counter the list to the global
4784          * container counter list. The list changes while query starts. In
4785          * this case, lock will not be needed as query callback and release
4786          * function both operate with the different list.
4787          *
4788          */
4789         if (!priv->counter_fallback)
4790                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4791         else
4792                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4793                                   (priv->sh, 0, 0))->counters),
4794                                   cnt, next);
4795 }
4796
4797 /**
4798  * Verify the @p attributes will be correctly understood by the NIC and store
4799  * them in the @p flow if everything is correct.
4800  *
4801  * @param[in] dev
4802  *   Pointer to dev struct.
4803  * @param[in] attributes
4804  *   Pointer to flow attributes
4805  * @param[in] external
4806  *   This flow rule is created by request external to PMD.
4807  * @param[out] error
4808  *   Pointer to error structure.
4809  *
4810  * @return
4811  *   - 0 on success and non root table.
4812  *   - 1 on success and root table.
4813  *   - a negative errno value otherwise and rte_errno is set.
4814  */
4815 static int
4816 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4817                             const struct rte_flow_attr *attributes,
4818                             bool external __rte_unused,
4819                             struct rte_flow_error *error)
4820 {
4821         struct mlx5_priv *priv = dev->data->dev_private;
4822         uint32_t priority_max = priv->config.flow_prio - 1;
4823         int ret = 0;
4824
4825 #ifndef HAVE_MLX5DV_DR
4826         if (attributes->group)
4827                 return rte_flow_error_set(error, ENOTSUP,
4828                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4829                                           NULL,
4830                                           "groups are not supported");
4831 #else
4832         uint32_t table = 0;
4833
4834         ret = mlx5_flow_group_to_table(attributes, external,
4835                                        attributes->group, !!priv->fdb_def_rule,
4836                                        &table, error);
4837         if (ret)
4838                 return ret;
4839         if (!table)
4840                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4841 #endif
4842         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4843             attributes->priority >= priority_max)
4844                 return rte_flow_error_set(error, ENOTSUP,
4845                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4846                                           NULL,
4847                                           "priority out of range");
4848         if (attributes->transfer) {
4849                 if (!priv->config.dv_esw_en)
4850                         return rte_flow_error_set
4851                                 (error, ENOTSUP,
4852                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4853                                  "E-Switch dr is not supported");
4854                 if (!(priv->representor || priv->master))
4855                         return rte_flow_error_set
4856                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4857                                  NULL, "E-Switch configuration can only be"
4858                                  " done by a master or a representor device");
4859                 if (attributes->egress)
4860                         return rte_flow_error_set
4861                                 (error, ENOTSUP,
4862                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4863                                  "egress is not supported");
4864         }
4865         if (!(attributes->egress ^ attributes->ingress))
4866                 return rte_flow_error_set(error, ENOTSUP,
4867                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4868                                           "must specify exactly one of "
4869                                           "ingress or egress");
4870         return ret;
4871 }
4872
4873 /**
4874  * Internal validation function. For validating both actions and items.
4875  *
4876  * @param[in] dev
4877  *   Pointer to the rte_eth_dev structure.
4878  * @param[in] attr
4879  *   Pointer to the flow attributes.
4880  * @param[in] items
4881  *   Pointer to the list of items.
4882  * @param[in] actions
4883  *   Pointer to the list of actions.
4884  * @param[in] external
4885  *   This flow rule is created by request external to PMD.
4886  * @param[in] hairpin
4887  *   Number of hairpin TX actions, 0 means classic flow.
4888  * @param[out] error
4889  *   Pointer to the error structure.
4890  *
4891  * @return
4892  *   0 on success, a negative errno value otherwise and rte_errno is set.
4893  */
4894 static int
4895 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4896                  const struct rte_flow_item items[],
4897                  const struct rte_flow_action actions[],
4898                  bool external, int hairpin, struct rte_flow_error *error)
4899 {
4900         int ret;
4901         uint64_t action_flags = 0;
4902         uint64_t item_flags = 0;
4903         uint64_t last_item = 0;
4904         uint8_t next_protocol = 0xff;
4905         uint16_t ether_type = 0;
4906         int actions_n = 0;
4907         uint8_t item_ipv6_proto = 0;
4908         const struct rte_flow_item *gre_item = NULL;
4909         const struct rte_flow_action_raw_decap *decap;
4910         const struct rte_flow_action_raw_encap *encap;
4911         const struct rte_flow_action_rss *rss;
4912         const struct rte_flow_item_tcp nic_tcp_mask = {
4913                 .hdr = {
4914                         .tcp_flags = 0xFF,
4915                         .src_port = RTE_BE16(UINT16_MAX),
4916                         .dst_port = RTE_BE16(UINT16_MAX),
4917                 }
4918         };
4919         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4920                 .hdr = {
4921                         .src_addr = RTE_BE32(0xffffffff),
4922                         .dst_addr = RTE_BE32(0xffffffff),
4923                         .type_of_service = 0xff,
4924                         .next_proto_id = 0xff,
4925                         .time_to_live = 0xff,
4926                 },
4927         };
4928         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4929                 .hdr = {
4930                         .src_addr =
4931                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4932                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4933                         .dst_addr =
4934                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4935                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4936                         .vtc_flow = RTE_BE32(0xffffffff),
4937                         .proto = 0xff,
4938                         .hop_limits = 0xff,
4939                 },
4940         };
4941         const struct rte_flow_item_ecpri nic_ecpri_mask = {
4942                 .hdr = {
4943                         .common = {
4944                                 .u32 =
4945                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
4946                                         .type = 0xFF,
4947                                         }).u32),
4948                         },
4949                         .dummy[0] = 0xffffffff,
4950                 },
4951         };
4952         struct mlx5_priv *priv = dev->data->dev_private;
4953         struct mlx5_dev_config *dev_conf = &priv->config;
4954         uint16_t queue_index = 0xFFFF;
4955         const struct rte_flow_item_vlan *vlan_m = NULL;
4956         int16_t rw_act_num = 0;
4957         uint64_t is_root;
4958
4959         if (items == NULL)
4960                 return -1;
4961         ret = flow_dv_validate_attributes(dev, attr, external, error);
4962         if (ret < 0)
4963                 return ret;
4964         is_root = (uint64_t)ret;
4965         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4966                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4967                 int type = items->type;
4968
4969                 if (!mlx5_flow_os_item_supported(type))
4970                         return rte_flow_error_set(error, ENOTSUP,
4971                                                   RTE_FLOW_ERROR_TYPE_ITEM,
4972                                                   NULL, "item not supported");
4973                 switch (type) {
4974                 case RTE_FLOW_ITEM_TYPE_VOID:
4975                         break;
4976                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4977                         ret = flow_dv_validate_item_port_id
4978                                         (dev, items, attr, item_flags, error);
4979                         if (ret < 0)
4980                                 return ret;
4981                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4982                         break;
4983                 case RTE_FLOW_ITEM_TYPE_ETH:
4984                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4985                                                           error);
4986                         if (ret < 0)
4987                                 return ret;
4988                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4989                                              MLX5_FLOW_LAYER_OUTER_L2;
4990                         if (items->mask != NULL && items->spec != NULL) {
4991                                 ether_type =
4992                                         ((const struct rte_flow_item_eth *)
4993                                          items->spec)->type;
4994                                 ether_type &=
4995                                         ((const struct rte_flow_item_eth *)
4996                                          items->mask)->type;
4997                                 ether_type = rte_be_to_cpu_16(ether_type);
4998                         } else {
4999                                 ether_type = 0;
5000                         }
5001                         break;
5002                 case RTE_FLOW_ITEM_TYPE_VLAN:
5003                         ret = flow_dv_validate_item_vlan(items, item_flags,
5004                                                          dev, error);
5005                         if (ret < 0)
5006                                 return ret;
5007                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5008                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5009                         if (items->mask != NULL && items->spec != NULL) {
5010                                 ether_type =
5011                                         ((const struct rte_flow_item_vlan *)
5012                                          items->spec)->inner_type;
5013                                 ether_type &=
5014                                         ((const struct rte_flow_item_vlan *)
5015                                          items->mask)->inner_type;
5016                                 ether_type = rte_be_to_cpu_16(ether_type);
5017                         } else {
5018                                 ether_type = 0;
5019                         }
5020                         /* Store outer VLAN mask for of_push_vlan action. */
5021                         if (!tunnel)
5022                                 vlan_m = items->mask;
5023                         break;
5024                 case RTE_FLOW_ITEM_TYPE_IPV4:
5025                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5026                                                   &item_flags, &tunnel);
5027                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
5028                                                            last_item,
5029                                                            ether_type,
5030                                                            &nic_ipv4_mask,
5031                                                            error);
5032                         if (ret < 0)
5033                                 return ret;
5034                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5035                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5036                         if (items->mask != NULL &&
5037                             ((const struct rte_flow_item_ipv4 *)
5038                              items->mask)->hdr.next_proto_id) {
5039                                 next_protocol =
5040                                         ((const struct rte_flow_item_ipv4 *)
5041                                          (items->spec))->hdr.next_proto_id;
5042                                 next_protocol &=
5043                                         ((const struct rte_flow_item_ipv4 *)
5044                                          (items->mask))->hdr.next_proto_id;
5045                         } else {
5046                                 /* Reset for inner layer. */
5047                                 next_protocol = 0xff;
5048                         }
5049                         break;
5050                 case RTE_FLOW_ITEM_TYPE_IPV6:
5051                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5052                                                   &item_flags, &tunnel);
5053                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5054                                                            last_item,
5055                                                            ether_type,
5056                                                            &nic_ipv6_mask,
5057                                                            error);
5058                         if (ret < 0)
5059                                 return ret;
5060                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5061                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5062                         if (items->mask != NULL &&
5063                             ((const struct rte_flow_item_ipv6 *)
5064                              items->mask)->hdr.proto) {
5065                                 item_ipv6_proto =
5066                                         ((const struct rte_flow_item_ipv6 *)
5067                                          items->spec)->hdr.proto;
5068                                 next_protocol =
5069                                         ((const struct rte_flow_item_ipv6 *)
5070                                          items->spec)->hdr.proto;
5071                                 next_protocol &=
5072                                         ((const struct rte_flow_item_ipv6 *)
5073                                          items->mask)->hdr.proto;
5074                         } else {
5075                                 /* Reset for inner layer. */
5076                                 next_protocol = 0xff;
5077                         }
5078                         break;
5079                 case RTE_FLOW_ITEM_TYPE_TCP:
5080                         ret = mlx5_flow_validate_item_tcp
5081                                                 (items, item_flags,
5082                                                  next_protocol,
5083                                                  &nic_tcp_mask,
5084                                                  error);
5085                         if (ret < 0)
5086                                 return ret;
5087                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5088                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5089                         break;
5090                 case RTE_FLOW_ITEM_TYPE_UDP:
5091                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5092                                                           next_protocol,
5093                                                           error);
5094                         if (ret < 0)
5095                                 return ret;
5096                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5097                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5098                         break;
5099                 case RTE_FLOW_ITEM_TYPE_GRE:
5100                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5101                                                           next_protocol, error);
5102                         if (ret < 0)
5103                                 return ret;
5104                         gre_item = items;
5105                         last_item = MLX5_FLOW_LAYER_GRE;
5106                         break;
5107                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5108                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5109                                                             next_protocol,
5110                                                             error);
5111                         if (ret < 0)
5112                                 return ret;
5113                         last_item = MLX5_FLOW_LAYER_NVGRE;
5114                         break;
5115                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5116                         ret = mlx5_flow_validate_item_gre_key
5117                                 (items, item_flags, gre_item, error);
5118                         if (ret < 0)
5119                                 return ret;
5120                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5121                         break;
5122                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5123                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5124                                                             error);
5125                         if (ret < 0)
5126                                 return ret;
5127                         last_item = MLX5_FLOW_LAYER_VXLAN;
5128                         break;
5129                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5130                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5131                                                                 item_flags, dev,
5132                                                                 error);
5133                         if (ret < 0)
5134                                 return ret;
5135                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5136                         break;
5137                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5138                         ret = mlx5_flow_validate_item_geneve(items,
5139                                                              item_flags, dev,
5140                                                              error);
5141                         if (ret < 0)
5142                                 return ret;
5143                         last_item = MLX5_FLOW_LAYER_GENEVE;
5144                         break;
5145                 case RTE_FLOW_ITEM_TYPE_MPLS:
5146                         ret = mlx5_flow_validate_item_mpls(dev, items,
5147                                                            item_flags,
5148                                                            last_item, error);
5149                         if (ret < 0)
5150                                 return ret;
5151                         last_item = MLX5_FLOW_LAYER_MPLS;
5152                         break;
5153
5154                 case RTE_FLOW_ITEM_TYPE_MARK:
5155                         ret = flow_dv_validate_item_mark(dev, items, attr,
5156                                                          error);
5157                         if (ret < 0)
5158                                 return ret;
5159                         last_item = MLX5_FLOW_ITEM_MARK;
5160                         break;
5161                 case RTE_FLOW_ITEM_TYPE_META:
5162                         ret = flow_dv_validate_item_meta(dev, items, attr,
5163                                                          error);
5164                         if (ret < 0)
5165                                 return ret;
5166                         last_item = MLX5_FLOW_ITEM_METADATA;
5167                         break;
5168                 case RTE_FLOW_ITEM_TYPE_ICMP:
5169                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5170                                                            next_protocol,
5171                                                            error);
5172                         if (ret < 0)
5173                                 return ret;
5174                         last_item = MLX5_FLOW_LAYER_ICMP;
5175                         break;
5176                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5177                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5178                                                             next_protocol,
5179                                                             error);
5180                         if (ret < 0)
5181                                 return ret;
5182                         item_ipv6_proto = IPPROTO_ICMPV6;
5183                         last_item = MLX5_FLOW_LAYER_ICMP6;
5184                         break;
5185                 case RTE_FLOW_ITEM_TYPE_TAG:
5186                         ret = flow_dv_validate_item_tag(dev, items,
5187                                                         attr, error);
5188                         if (ret < 0)
5189                                 return ret;
5190                         last_item = MLX5_FLOW_ITEM_TAG;
5191                         break;
5192                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5193                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5194                         break;
5195                 case RTE_FLOW_ITEM_TYPE_GTP:
5196                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5197                                                         error);
5198                         if (ret < 0)
5199                                 return ret;
5200                         last_item = MLX5_FLOW_LAYER_GTP;
5201                         break;
5202                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5203                         /* Capacity will be checked in the translate stage. */
5204                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5205                                                             last_item,
5206                                                             ether_type,
5207                                                             &nic_ecpri_mask,
5208                                                             error);
5209                         if (ret < 0)
5210                                 return ret;
5211                         last_item = MLX5_FLOW_LAYER_ECPRI;
5212                         break;
5213                 default:
5214                         return rte_flow_error_set(error, ENOTSUP,
5215                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5216                                                   NULL, "item not supported");
5217                 }
5218                 item_flags |= last_item;
5219         }
5220         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5221                 int type = actions->type;
5222
5223                 if (!mlx5_flow_os_action_supported(type))
5224                         return rte_flow_error_set(error, ENOTSUP,
5225                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5226                                                   actions,
5227                                                   "action not supported");
5228                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5229                         return rte_flow_error_set(error, ENOTSUP,
5230                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5231                                                   actions, "too many actions");
5232                 switch (type) {
5233                 case RTE_FLOW_ACTION_TYPE_VOID:
5234                         break;
5235                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5236                         ret = flow_dv_validate_action_port_id(dev,
5237                                                               action_flags,
5238                                                               actions,
5239                                                               attr,
5240                                                               error);
5241                         if (ret)
5242                                 return ret;
5243                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5244                         ++actions_n;
5245                         break;
5246                 case RTE_FLOW_ACTION_TYPE_FLAG:
5247                         ret = flow_dv_validate_action_flag(dev, action_flags,
5248                                                            attr, error);
5249                         if (ret < 0)
5250                                 return ret;
5251                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5252                                 /* Count all modify-header actions as one. */
5253                                 if (!(action_flags &
5254                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5255                                         ++actions_n;
5256                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5257                                                 MLX5_FLOW_ACTION_MARK_EXT;
5258                         } else {
5259                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5260                                 ++actions_n;
5261                         }
5262                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5263                         break;
5264                 case RTE_FLOW_ACTION_TYPE_MARK:
5265                         ret = flow_dv_validate_action_mark(dev, actions,
5266                                                            action_flags,
5267                                                            attr, error);
5268                         if (ret < 0)
5269                                 return ret;
5270                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5271                                 /* Count all modify-header actions as one. */
5272                                 if (!(action_flags &
5273                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5274                                         ++actions_n;
5275                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5276                                                 MLX5_FLOW_ACTION_MARK_EXT;
5277                         } else {
5278                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5279                                 ++actions_n;
5280                         }
5281                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5282                         break;
5283                 case RTE_FLOW_ACTION_TYPE_SET_META:
5284                         ret = flow_dv_validate_action_set_meta(dev, actions,
5285                                                                action_flags,
5286                                                                attr, error);
5287                         if (ret < 0)
5288                                 return ret;
5289                         /* Count all modify-header actions as one action. */
5290                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5291                                 ++actions_n;
5292                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5293                         rw_act_num += MLX5_ACT_NUM_SET_META;
5294                         break;
5295                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5296                         ret = flow_dv_validate_action_set_tag(dev, actions,
5297                                                               action_flags,
5298                                                               attr, error);
5299                         if (ret < 0)
5300                                 return ret;
5301                         /* Count all modify-header actions as one action. */
5302                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5303                                 ++actions_n;
5304                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5305                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5306                         break;
5307                 case RTE_FLOW_ACTION_TYPE_DROP:
5308                         ret = mlx5_flow_validate_action_drop(action_flags,
5309                                                              attr, error);
5310                         if (ret < 0)
5311                                 return ret;
5312                         action_flags |= MLX5_FLOW_ACTION_DROP;
5313                         ++actions_n;
5314                         break;
5315                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5316                         ret = mlx5_flow_validate_action_queue(actions,
5317                                                               action_flags, dev,
5318                                                               attr, error);
5319                         if (ret < 0)
5320                                 return ret;
5321                         queue_index = ((const struct rte_flow_action_queue *)
5322                                                         (actions->conf))->index;
5323                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5324                         ++actions_n;
5325                         break;
5326                 case RTE_FLOW_ACTION_TYPE_RSS:
5327                         rss = actions->conf;
5328                         ret = mlx5_flow_validate_action_rss(actions,
5329                                                             action_flags, dev,
5330                                                             attr, item_flags,
5331                                                             error);
5332                         if (ret < 0)
5333                                 return ret;
5334                         if (rss != NULL && rss->queue_num)
5335                                 queue_index = rss->queue[0];
5336                         action_flags |= MLX5_FLOW_ACTION_RSS;
5337                         ++actions_n;
5338                         break;
5339                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5340                         ret =
5341                         mlx5_flow_validate_action_default_miss(action_flags,
5342                                         attr, error);
5343                         if (ret < 0)
5344                                 return ret;
5345                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5346                         ++actions_n;
5347                         break;
5348                 case RTE_FLOW_ACTION_TYPE_COUNT:
5349                         ret = flow_dv_validate_action_count(dev, error);
5350                         if (ret < 0)
5351                                 return ret;
5352                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5353                         ++actions_n;
5354                         break;
5355                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5356                         if (flow_dv_validate_action_pop_vlan(dev,
5357                                                              action_flags,
5358                                                              actions,
5359                                                              item_flags, attr,
5360                                                              error))
5361                                 return -rte_errno;
5362                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5363                         ++actions_n;
5364                         break;
5365                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5366                         ret = flow_dv_validate_action_push_vlan(dev,
5367                                                                 action_flags,
5368                                                                 vlan_m,
5369                                                                 actions, attr,
5370                                                                 error);
5371                         if (ret < 0)
5372                                 return ret;
5373                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5374                         ++actions_n;
5375                         break;
5376                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5377                         ret = flow_dv_validate_action_set_vlan_pcp
5378                                                 (action_flags, actions, error);
5379                         if (ret < 0)
5380                                 return ret;
5381                         /* Count PCP with push_vlan command. */
5382                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5383                         break;
5384                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5385                         ret = flow_dv_validate_action_set_vlan_vid
5386                                                 (item_flags, action_flags,
5387                                                  actions, error);
5388                         if (ret < 0)
5389                                 return ret;
5390                         /* Count VID with push_vlan command. */
5391                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5392                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5393                         break;
5394                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5395                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5396                         ret = flow_dv_validate_action_l2_encap(dev,
5397                                                                action_flags,
5398                                                                actions, attr,
5399                                                                error);
5400                         if (ret < 0)
5401                                 return ret;
5402                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5403                         ++actions_n;
5404                         break;
5405                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5406                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5407                         ret = flow_dv_validate_action_decap(dev, action_flags,
5408                                                             attr, error);
5409                         if (ret < 0)
5410                                 return ret;
5411                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5412                         ++actions_n;
5413                         break;
5414                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5415                         ret = flow_dv_validate_action_raw_encap_decap
5416                                 (dev, NULL, actions->conf, attr, &action_flags,
5417                                  &actions_n, error);
5418                         if (ret < 0)
5419                                 return ret;
5420                         break;
5421                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5422                         decap = actions->conf;
5423                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5424                                 ;
5425                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5426                                 encap = NULL;
5427                                 actions--;
5428                         } else {
5429                                 encap = actions->conf;
5430                         }
5431                         ret = flow_dv_validate_action_raw_encap_decap
5432                                            (dev,
5433                                             decap ? decap : &empty_decap, encap,
5434                                             attr, &action_flags, &actions_n,
5435                                             error);
5436                         if (ret < 0)
5437                                 return ret;
5438                         break;
5439                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5440                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5441                         ret = flow_dv_validate_action_modify_mac(action_flags,
5442                                                                  actions,
5443                                                                  item_flags,
5444                                                                  error);
5445                         if (ret < 0)
5446                                 return ret;
5447                         /* Count all modify-header actions as one action. */
5448                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5449                                 ++actions_n;
5450                         action_flags |= actions->type ==
5451                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5452                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5453                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5454                         /*
5455                          * Even if the source and destination MAC addresses have
5456                          * overlap in the header with 4B alignment, the convert
5457                          * function will handle them separately and 4 SW actions
5458                          * will be created. And 2 actions will be added each
5459                          * time no matter how many bytes of address will be set.
5460                          */
5461                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5462                         break;
5463                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5464                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5465                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5466                                                                   actions,
5467                                                                   item_flags,
5468                                                                   error);
5469                         if (ret < 0)
5470                                 return ret;
5471                         /* Count all modify-header actions as one action. */
5472                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5473                                 ++actions_n;
5474                         action_flags |= actions->type ==
5475                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5476                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5477                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5478                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5479                         break;
5480                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5481                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5482                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5483                                                                   actions,
5484                                                                   item_flags,
5485                                                                   error);
5486                         if (ret < 0)
5487                                 return ret;
5488                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5489                                 return rte_flow_error_set(error, ENOTSUP,
5490                                         RTE_FLOW_ERROR_TYPE_ACTION,
5491                                         actions,
5492                                         "Can't change header "
5493                                         "with ICMPv6 proto");
5494                         /* Count all modify-header actions as one action. */
5495                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5496                                 ++actions_n;
5497                         action_flags |= actions->type ==
5498                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5499                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5500                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5501                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5502                         break;
5503                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5504                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5505                         ret = flow_dv_validate_action_modify_tp(action_flags,
5506                                                                 actions,
5507                                                                 item_flags,
5508                                                                 error);
5509                         if (ret < 0)
5510                                 return ret;
5511                         /* Count all modify-header actions as one action. */
5512                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5513                                 ++actions_n;
5514                         action_flags |= actions->type ==
5515                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5516                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5517                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5518                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5519                         break;
5520                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5521                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5522                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5523                                                                  actions,
5524                                                                  item_flags,
5525                                                                  error);
5526                         if (ret < 0)
5527                                 return ret;
5528                         /* Count all modify-header actions as one action. */
5529                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5530                                 ++actions_n;
5531                         action_flags |= actions->type ==
5532                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5533                                                 MLX5_FLOW_ACTION_SET_TTL :
5534                                                 MLX5_FLOW_ACTION_DEC_TTL;
5535                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5536                         break;
5537                 case RTE_FLOW_ACTION_TYPE_JUMP:
5538                         ret = flow_dv_validate_action_jump(actions,
5539                                                            action_flags,
5540                                                            attr, external,
5541                                                            error);
5542                         if (ret)
5543                                 return ret;
5544                         ++actions_n;
5545                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5546                         break;
5547                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5548                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5549                         ret = flow_dv_validate_action_modify_tcp_seq
5550                                                                 (action_flags,
5551                                                                  actions,
5552                                                                  item_flags,
5553                                                                  error);
5554                         if (ret < 0)
5555                                 return ret;
5556                         /* Count all modify-header actions as one action. */
5557                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5558                                 ++actions_n;
5559                         action_flags |= actions->type ==
5560                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5561                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5562                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5563                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5564                         break;
5565                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5566                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5567                         ret = flow_dv_validate_action_modify_tcp_ack
5568                                                                 (action_flags,
5569                                                                  actions,
5570                                                                  item_flags,
5571                                                                  error);
5572                         if (ret < 0)
5573                                 return ret;
5574                         /* Count all modify-header actions as one action. */
5575                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5576                                 ++actions_n;
5577                         action_flags |= actions->type ==
5578                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5579                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5580                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5581                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5582                         break;
5583                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5584                         break;
5585                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5586                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5587                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5588                         break;
5589                 case RTE_FLOW_ACTION_TYPE_METER:
5590                         ret = mlx5_flow_validate_action_meter(dev,
5591                                                               action_flags,
5592                                                               actions, attr,
5593                                                               error);
5594                         if (ret < 0)
5595                                 return ret;
5596                         action_flags |= MLX5_FLOW_ACTION_METER;
5597                         ++actions_n;
5598                         /* Meter action will add one more TAG action. */
5599                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5600                         break;
5601                 case RTE_FLOW_ACTION_TYPE_AGE:
5602                         ret = flow_dv_validate_action_age(action_flags,
5603                                                           actions, dev,
5604                                                           error);
5605                         if (ret < 0)
5606                                 return ret;
5607                         action_flags |= MLX5_FLOW_ACTION_AGE;
5608                         ++actions_n;
5609                         break;
5610                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5611                         ret = flow_dv_validate_action_modify_ipv4_dscp
5612                                                          (action_flags,
5613                                                           actions,
5614                                                           item_flags,
5615                                                           error);
5616                         if (ret < 0)
5617                                 return ret;
5618                         /* Count all modify-header actions as one action. */
5619                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5620                                 ++actions_n;
5621                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5622                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5623                         break;
5624                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5625                         ret = flow_dv_validate_action_modify_ipv6_dscp
5626                                                                 (action_flags,
5627                                                                  actions,
5628                                                                  item_flags,
5629                                                                  error);
5630                         if (ret < 0)
5631                                 return ret;
5632                         /* Count all modify-header actions as one action. */
5633                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5634                                 ++actions_n;
5635                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5636                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5637                         break;
5638                 default:
5639                         return rte_flow_error_set(error, ENOTSUP,
5640                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5641                                                   actions,
5642                                                   "action not supported");
5643                 }
5644         }
5645         /*
5646          * Validate the drop action mutual exclusion with other actions.
5647          * Drop action is mutually-exclusive with any other action, except for
5648          * Count action.
5649          */
5650         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5651             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5652                 return rte_flow_error_set(error, EINVAL,
5653                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5654                                           "Drop action is mutually-exclusive "
5655                                           "with any other action, except for "
5656                                           "Count action");
5657         /* Eswitch has few restrictions on using items and actions */
5658         if (attr->transfer) {
5659                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5660                     action_flags & MLX5_FLOW_ACTION_FLAG)
5661                         return rte_flow_error_set(error, ENOTSUP,
5662                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5663                                                   NULL,
5664                                                   "unsupported action FLAG");
5665                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5666                     action_flags & MLX5_FLOW_ACTION_MARK)
5667                         return rte_flow_error_set(error, ENOTSUP,
5668                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5669                                                   NULL,
5670                                                   "unsupported action MARK");
5671                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5672                         return rte_flow_error_set(error, ENOTSUP,
5673                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5674                                                   NULL,
5675                                                   "unsupported action QUEUE");
5676                 if (action_flags & MLX5_FLOW_ACTION_RSS)
5677                         return rte_flow_error_set(error, ENOTSUP,
5678                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5679                                                   NULL,
5680                                                   "unsupported action RSS");
5681                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5682                         return rte_flow_error_set(error, EINVAL,
5683                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5684                                                   actions,
5685                                                   "no fate action is found");
5686         } else {
5687                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5688                         return rte_flow_error_set(error, EINVAL,
5689                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5690                                                   actions,
5691                                                   "no fate action is found");
5692         }
5693         /* Continue validation for Xcap actions.*/
5694         if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5695             mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5696                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5697                     MLX5_FLOW_XCAP_ACTIONS)
5698                         return rte_flow_error_set(error, ENOTSUP,
5699                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5700                                                   NULL, "encap and decap "
5701                                                   "combination aren't supported");
5702                 if (!attr->transfer && attr->ingress && (action_flags &
5703                                                         MLX5_FLOW_ACTION_ENCAP))
5704                         return rte_flow_error_set(error, ENOTSUP,
5705                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5706                                                   NULL, "encap is not supported"
5707                                                   " for ingress traffic");
5708         }
5709         /* Hairpin flow will add one more TAG action. */
5710         if (hairpin > 0)
5711                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5712         /* extra metadata enabled: one more TAG action will be add. */
5713         if (dev_conf->dv_flow_en &&
5714             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5715             mlx5_flow_ext_mreg_supported(dev))
5716                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5717         if ((uint32_t)rw_act_num >
5718                         flow_dv_modify_hdr_action_max(dev, is_root)) {
5719                 return rte_flow_error_set(error, ENOTSUP,
5720                                           RTE_FLOW_ERROR_TYPE_ACTION,
5721                                           NULL, "too many header modify"
5722                                           " actions to support");
5723         }
5724         return 0;
5725 }
5726
5727 /**
5728  * Internal preparation function. Allocates the DV flow size,
5729  * this size is constant.
5730  *
5731  * @param[in] dev
5732  *   Pointer to the rte_eth_dev structure.
5733  * @param[in] attr
5734  *   Pointer to the flow attributes.
5735  * @param[in] items
5736  *   Pointer to the list of items.
5737  * @param[in] actions
5738  *   Pointer to the list of actions.
5739  * @param[out] error
5740  *   Pointer to the error structure.
5741  *
5742  * @return
5743  *   Pointer to mlx5_flow object on success,
5744  *   otherwise NULL and rte_errno is set.
5745  */
5746 static struct mlx5_flow *
5747 flow_dv_prepare(struct rte_eth_dev *dev,
5748                 const struct rte_flow_attr *attr __rte_unused,
5749                 const struct rte_flow_item items[] __rte_unused,
5750                 const struct rte_flow_action actions[] __rte_unused,
5751                 struct rte_flow_error *error)
5752 {
5753         uint32_t handle_idx = 0;
5754         struct mlx5_flow *dev_flow;
5755         struct mlx5_flow_handle *dev_handle;
5756         struct mlx5_priv *priv = dev->data->dev_private;
5757
5758         /* In case of corrupting the memory. */
5759         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5760                 rte_flow_error_set(error, ENOSPC,
5761                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5762                                    "not free temporary device flow");
5763                 return NULL;
5764         }
5765         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5766                                    &handle_idx);
5767         if (!dev_handle) {
5768                 rte_flow_error_set(error, ENOMEM,
5769                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5770                                    "not enough memory to create flow handle");
5771                 return NULL;
5772         }
5773         /* No multi-thread supporting. */
5774         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5775         dev_flow->handle = dev_handle;
5776         dev_flow->handle_idx = handle_idx;
5777         /*
5778          * In some old rdma-core releases, before continuing, a check of the
5779          * length of matching parameter will be done at first. It needs to use
5780          * the length without misc4 param. If the flow has misc4 support, then
5781          * the length needs to be adjusted accordingly. Each param member is
5782          * aligned with a 64B boundary naturally.
5783          */
5784         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
5785                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
5786         /*
5787          * The matching value needs to be cleared to 0 before using. In the
5788          * past, it will be automatically cleared when using rte_*alloc
5789          * API. The time consumption will be almost the same as before.
5790          */
5791         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5792         dev_flow->ingress = attr->ingress;
5793         dev_flow->dv.transfer = attr->transfer;
5794         return dev_flow;
5795 }
5796
5797 #ifdef RTE_LIBRTE_MLX5_DEBUG
5798 /**
5799  * Sanity check for match mask and value. Similar to check_valid_spec() in
5800  * kernel driver. If unmasked bit is present in value, it returns failure.
5801  *
5802  * @param match_mask
5803  *   pointer to match mask buffer.
5804  * @param match_value
5805  *   pointer to match value buffer.
5806  *
5807  * @return
5808  *   0 if valid, -EINVAL otherwise.
5809  */
5810 static int
5811 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5812 {
5813         uint8_t *m = match_mask;
5814         uint8_t *v = match_value;
5815         unsigned int i;
5816
5817         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5818                 if (v[i] & ~m[i]) {
5819                         DRV_LOG(ERR,
5820                                 "match_value differs from match_criteria"
5821                                 " %p[%u] != %p[%u]",
5822                                 match_value, i, match_mask, i);
5823                         return -EINVAL;
5824                 }
5825         }
5826         return 0;
5827 }
5828 #endif
5829
5830 /**
5831  * Add match of ip_version.
5832  *
5833  * @param[in] group
5834  *   Flow group.
5835  * @param[in] headers_v
5836  *   Values header pointer.
5837  * @param[in] headers_m
5838  *   Masks header pointer.
5839  * @param[in] ip_version
5840  *   The IP version to set.
5841  */
5842 static inline void
5843 flow_dv_set_match_ip_version(uint32_t group,
5844                              void *headers_v,
5845                              void *headers_m,
5846                              uint8_t ip_version)
5847 {
5848         if (group == 0)
5849                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5850         else
5851                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5852                          ip_version);
5853         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5854         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5855         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5856 }
5857
5858 /**
5859  * Add Ethernet item to matcher and to the value.
5860  *
5861  * @param[in, out] matcher
5862  *   Flow matcher.
5863  * @param[in, out] key
5864  *   Flow matcher value.
5865  * @param[in] item
5866  *   Flow pattern to translate.
5867  * @param[in] inner
5868  *   Item is inner pattern.
5869  */
5870 static void
5871 flow_dv_translate_item_eth(void *matcher, void *key,
5872                            const struct rte_flow_item *item, int inner,
5873                            uint32_t group)
5874 {
5875         const struct rte_flow_item_eth *eth_m = item->mask;
5876         const struct rte_flow_item_eth *eth_v = item->spec;
5877         const struct rte_flow_item_eth nic_mask = {
5878                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5879                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5880                 .type = RTE_BE16(0xffff),
5881         };
5882         void *headers_m;
5883         void *headers_v;
5884         char *l24_v;
5885         unsigned int i;
5886
5887         if (!eth_v)
5888                 return;
5889         if (!eth_m)
5890                 eth_m = &nic_mask;
5891         if (inner) {
5892                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5893                                          inner_headers);
5894                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5895         } else {
5896                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5897                                          outer_headers);
5898                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5899         }
5900         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5901                &eth_m->dst, sizeof(eth_m->dst));
5902         /* The value must be in the range of the mask. */
5903         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5904         for (i = 0; i < sizeof(eth_m->dst); ++i)
5905                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5906         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5907                &eth_m->src, sizeof(eth_m->src));
5908         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5909         /* The value must be in the range of the mask. */
5910         for (i = 0; i < sizeof(eth_m->dst); ++i)
5911                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5912         if (eth_v->type) {
5913                 /* When ethertype is present set mask for tagged VLAN. */
5914                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5915                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5916                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5917                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5918                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5919                                  1);
5920                         /* Return here to avoid setting match on ethertype. */
5921                         return;
5922                 }
5923         }
5924         /*
5925          * HW supports match on one Ethertype, the Ethertype following the last
5926          * VLAN tag of the packet (see PRM).
5927          * Set match on ethertype only if ETH header is not followed by VLAN.
5928          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5929          * ethertype, and use ip_version field instead.
5930          * eCPRI over Ether layer will use type value 0xAEFE.
5931          */
5932         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5933             eth_m->type == 0xFFFF) {
5934                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5935         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5936                    eth_m->type == 0xFFFF) {
5937                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5938         } else {
5939                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5940                          rte_be_to_cpu_16(eth_m->type));
5941                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5942                                      ethertype);
5943                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5944         }
5945 }
5946
5947 /**
5948  * Add VLAN item to matcher and to the value.
5949  *
5950  * @param[in, out] dev_flow
5951  *   Flow descriptor.
5952  * @param[in, out] matcher
5953  *   Flow matcher.
5954  * @param[in, out] key
5955  *   Flow matcher value.
5956  * @param[in] item
5957  *   Flow pattern to translate.
5958  * @param[in] inner
5959  *   Item is inner pattern.
5960  */
5961 static void
5962 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5963                             void *matcher, void *key,
5964                             const struct rte_flow_item *item,
5965                             int inner, uint32_t group)
5966 {
5967         const struct rte_flow_item_vlan *vlan_m = item->mask;
5968         const struct rte_flow_item_vlan *vlan_v = item->spec;
5969         void *headers_m;
5970         void *headers_v;
5971         uint16_t tci_m;
5972         uint16_t tci_v;
5973
5974         if (inner) {
5975                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5976                                          inner_headers);
5977                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5978         } else {
5979                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5980                                          outer_headers);
5981                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5982                 /*
5983                  * This is workaround, masks are not supported,
5984                  * and pre-validated.
5985                  */
5986                 if (vlan_v)
5987                         dev_flow->handle->vf_vlan.tag =
5988                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5989         }
5990         /*
5991          * When VLAN item exists in flow, mark packet as tagged,
5992          * even if TCI is not specified.
5993          */
5994         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5995         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5996         if (!vlan_v)
5997                 return;
5998         if (!vlan_m)
5999                 vlan_m = &rte_flow_item_vlan_mask;
6000         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6001         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6002         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6003         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6004         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6005         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6006         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6007         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6008         /*
6009          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6010          * ethertype, and use ip_version field instead.
6011          */
6012         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6013             vlan_m->inner_type == 0xFFFF) {
6014                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6015         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6016                    vlan_m->inner_type == 0xFFFF) {
6017                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6018         } else {
6019                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6020                          rte_be_to_cpu_16(vlan_m->inner_type));
6021                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6022                          rte_be_to_cpu_16(vlan_m->inner_type &
6023                                           vlan_v->inner_type));
6024         }
6025 }
6026
6027 /**
6028  * Add IPV4 item to matcher and to the value.
6029  *
6030  * @param[in, out] matcher
6031  *   Flow matcher.
6032  * @param[in, out] key
6033  *   Flow matcher value.
6034  * @param[in] item
6035  *   Flow pattern to translate.
6036  * @param[in] item_flags
6037  *   Bit-fields that holds the items detected until now.
6038  * @param[in] inner
6039  *   Item is inner pattern.
6040  * @param[in] group
6041  *   The group to insert the rule.
6042  */
6043 static void
6044 flow_dv_translate_item_ipv4(void *matcher, void *key,
6045                             const struct rte_flow_item *item,
6046                             const uint64_t item_flags,
6047                             int inner, uint32_t group)
6048 {
6049         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6050         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6051         const struct rte_flow_item_ipv4 nic_mask = {
6052                 .hdr = {
6053                         .src_addr = RTE_BE32(0xffffffff),
6054                         .dst_addr = RTE_BE32(0xffffffff),
6055                         .type_of_service = 0xff,
6056                         .next_proto_id = 0xff,
6057                         .time_to_live = 0xff,
6058                 },
6059         };
6060         void *headers_m;
6061         void *headers_v;
6062         char *l24_m;
6063         char *l24_v;
6064         uint8_t tos;
6065
6066         if (inner) {
6067                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6068                                          inner_headers);
6069                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6070         } else {
6071                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6072                                          outer_headers);
6073                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6074         }
6075         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6076         /*
6077          * On outer header (which must contains L2), or inner header with L2,
6078          * set cvlan_tag mask bit to mark this packet as untagged.
6079          * This should be done even if item->spec is empty.
6080          */
6081         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6082                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6083         if (!ipv4_v)
6084                 return;
6085         if (!ipv4_m)
6086                 ipv4_m = &nic_mask;
6087         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6088                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6089         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6090                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6091         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6092         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6093         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6094                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6095         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6096                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6097         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6098         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6099         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6100         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6101                  ipv4_m->hdr.type_of_service);
6102         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6103         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6104                  ipv4_m->hdr.type_of_service >> 2);
6105         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6106         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6107                  ipv4_m->hdr.next_proto_id);
6108         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6109                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6110         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6111                  ipv4_m->hdr.time_to_live);
6112         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6113                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6114 }
6115
6116 /**
6117  * Add IPV6 item to matcher and to the value.
6118  *
6119  * @param[in, out] matcher
6120  *   Flow matcher.
6121  * @param[in, out] key
6122  *   Flow matcher value.
6123  * @param[in] item
6124  *   Flow pattern to translate.
6125  * @param[in] item_flags
6126  *   Bit-fields that holds the items detected until now.
6127  * @param[in] inner
6128  *   Item is inner pattern.
6129  * @param[in] group
6130  *   The group to insert the rule.
6131  */
6132 static void
6133 flow_dv_translate_item_ipv6(void *matcher, void *key,
6134                             const struct rte_flow_item *item,
6135                             const uint64_t item_flags,
6136                             int inner, uint32_t group)
6137 {
6138         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6139         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6140         const struct rte_flow_item_ipv6 nic_mask = {
6141                 .hdr = {
6142                         .src_addr =
6143                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6144                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6145                         .dst_addr =
6146                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6147                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6148                         .vtc_flow = RTE_BE32(0xffffffff),
6149                         .proto = 0xff,
6150                         .hop_limits = 0xff,
6151                 },
6152         };
6153         void *headers_m;
6154         void *headers_v;
6155         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6156         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6157         char *l24_m;
6158         char *l24_v;
6159         uint32_t vtc_m;
6160         uint32_t vtc_v;
6161         int i;
6162         int size;
6163
6164         if (inner) {
6165                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6166                                          inner_headers);
6167                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6168         } else {
6169                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6170                                          outer_headers);
6171                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6172         }
6173         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6174         /*
6175          * On outer header (which must contains L2), or inner header with L2,
6176          * set cvlan_tag mask bit to mark this packet as untagged.
6177          * This should be done even if item->spec is empty.
6178          */
6179         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6180                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6181         if (!ipv6_v)
6182                 return;
6183         if (!ipv6_m)
6184                 ipv6_m = &nic_mask;
6185         size = sizeof(ipv6_m->hdr.dst_addr);
6186         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6187                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6188         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6189                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6190         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6191         for (i = 0; i < size; ++i)
6192                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6193         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6194                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6195         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6196                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6197         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6198         for (i = 0; i < size; ++i)
6199                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6200         /* TOS. */
6201         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6202         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6203         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6204         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6205         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6206         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6207         /* Label. */
6208         if (inner) {
6209                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6210                          vtc_m);
6211                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6212                          vtc_v);
6213         } else {
6214                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6215                          vtc_m);
6216                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6217                          vtc_v);
6218         }
6219         /* Protocol. */
6220         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6221                  ipv6_m->hdr.proto);
6222         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6223                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6224         /* Hop limit. */
6225         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6226                  ipv6_m->hdr.hop_limits);
6227         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6228                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6229 }
6230
6231 /**
6232  * Add TCP item to matcher and to the value.
6233  *
6234  * @param[in, out] matcher
6235  *   Flow matcher.
6236  * @param[in, out] key
6237  *   Flow matcher value.
6238  * @param[in] item
6239  *   Flow pattern to translate.
6240  * @param[in] inner
6241  *   Item is inner pattern.
6242  */
6243 static void
6244 flow_dv_translate_item_tcp(void *matcher, void *key,
6245                            const struct rte_flow_item *item,
6246                            int inner)
6247 {
6248         const struct rte_flow_item_tcp *tcp_m = item->mask;
6249         const struct rte_flow_item_tcp *tcp_v = item->spec;
6250         void *headers_m;
6251         void *headers_v;
6252
6253         if (inner) {
6254                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6255                                          inner_headers);
6256                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6257         } else {
6258                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6259                                          outer_headers);
6260                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6261         }
6262         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6263         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6264         if (!tcp_v)
6265                 return;
6266         if (!tcp_m)
6267                 tcp_m = &rte_flow_item_tcp_mask;
6268         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6269                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6270         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6271                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6272         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6273                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6274         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6275                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6276         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6277                  tcp_m->hdr.tcp_flags);
6278         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6279                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6280 }
6281
6282 /**
6283  * Add UDP item to matcher and to the value.
6284  *
6285  * @param[in, out] matcher
6286  *   Flow matcher.
6287  * @param[in, out] key
6288  *   Flow matcher value.
6289  * @param[in] item
6290  *   Flow pattern to translate.
6291  * @param[in] inner
6292  *   Item is inner pattern.
6293  */
6294 static void
6295 flow_dv_translate_item_udp(void *matcher, void *key,
6296                            const struct rte_flow_item *item,
6297                            int inner)
6298 {
6299         const struct rte_flow_item_udp *udp_m = item->mask;
6300         const struct rte_flow_item_udp *udp_v = item->spec;
6301         void *headers_m;
6302         void *headers_v;
6303
6304         if (inner) {
6305                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6306                                          inner_headers);
6307                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6308         } else {
6309                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6310                                          outer_headers);
6311                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6312         }
6313         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6314         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6315         if (!udp_v)
6316                 return;
6317         if (!udp_m)
6318                 udp_m = &rte_flow_item_udp_mask;
6319         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6320                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6321         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6322                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6323         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6324                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6325         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6326                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6327 }
6328
6329 /**
6330  * Add GRE optional Key item to matcher and to the value.
6331  *
6332  * @param[in, out] matcher
6333  *   Flow matcher.
6334  * @param[in, out] key
6335  *   Flow matcher value.
6336  * @param[in] item
6337  *   Flow pattern to translate.
6338  * @param[in] inner
6339  *   Item is inner pattern.
6340  */
6341 static void
6342 flow_dv_translate_item_gre_key(void *matcher, void *key,
6343                                    const struct rte_flow_item *item)
6344 {
6345         const rte_be32_t *key_m = item->mask;
6346         const rte_be32_t *key_v = item->spec;
6347         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6348         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6349         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6350
6351         /* GRE K bit must be on and should already be validated */
6352         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6353         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6354         if (!key_v)
6355                 return;
6356         if (!key_m)
6357                 key_m = &gre_key_default_mask;
6358         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6359                  rte_be_to_cpu_32(*key_m) >> 8);
6360         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6361                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6362         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6363                  rte_be_to_cpu_32(*key_m) & 0xFF);
6364         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6365                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6366 }
6367
6368 /**
6369  * Add GRE item to matcher and to the value.
6370  *
6371  * @param[in, out] matcher
6372  *   Flow matcher.
6373  * @param[in, out] key
6374  *   Flow matcher value.
6375  * @param[in] item
6376  *   Flow pattern to translate.
6377  * @param[in] inner
6378  *   Item is inner pattern.
6379  */
6380 static void
6381 flow_dv_translate_item_gre(void *matcher, void *key,
6382                            const struct rte_flow_item *item,
6383                            int inner)
6384 {
6385         const struct rte_flow_item_gre *gre_m = item->mask;
6386         const struct rte_flow_item_gre *gre_v = item->spec;
6387         void *headers_m;
6388         void *headers_v;
6389         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6390         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6391         struct {
6392                 union {
6393                         __extension__
6394                         struct {
6395                                 uint16_t version:3;
6396                                 uint16_t rsvd0:9;
6397                                 uint16_t s_present:1;
6398                                 uint16_t k_present:1;
6399                                 uint16_t rsvd_bit1:1;
6400                                 uint16_t c_present:1;
6401                         };
6402                         uint16_t value;
6403                 };
6404         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6405
6406         if (inner) {
6407                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6408                                          inner_headers);
6409                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6410         } else {
6411                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6412                                          outer_headers);
6413                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6414         }
6415         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6416         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6417         if (!gre_v)
6418                 return;
6419         if (!gre_m)
6420                 gre_m = &rte_flow_item_gre_mask;
6421         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6422                  rte_be_to_cpu_16(gre_m->protocol));
6423         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6424                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6425         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6426         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6427         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6428                  gre_crks_rsvd0_ver_m.c_present);
6429         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6430                  gre_crks_rsvd0_ver_v.c_present &
6431                  gre_crks_rsvd0_ver_m.c_present);
6432         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6433                  gre_crks_rsvd0_ver_m.k_present);
6434         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6435                  gre_crks_rsvd0_ver_v.k_present &
6436                  gre_crks_rsvd0_ver_m.k_present);
6437         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6438                  gre_crks_rsvd0_ver_m.s_present);
6439         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6440                  gre_crks_rsvd0_ver_v.s_present &
6441                  gre_crks_rsvd0_ver_m.s_present);
6442 }
6443
6444 /**
6445  * Add NVGRE item to matcher and to the value.
6446  *
6447  * @param[in, out] matcher
6448  *   Flow matcher.
6449  * @param[in, out] key
6450  *   Flow matcher value.
6451  * @param[in] item
6452  *   Flow pattern to translate.
6453  * @param[in] inner
6454  *   Item is inner pattern.
6455  */
6456 static void
6457 flow_dv_translate_item_nvgre(void *matcher, void *key,
6458                              const struct rte_flow_item *item,
6459                              int inner)
6460 {
6461         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6462         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6463         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6464         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6465         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6466         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6467         char *gre_key_m;
6468         char *gre_key_v;
6469         int size;
6470         int i;
6471
6472         /* For NVGRE, GRE header fields must be set with defined values. */
6473         const struct rte_flow_item_gre gre_spec = {
6474                 .c_rsvd0_ver = RTE_BE16(0x2000),
6475                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6476         };
6477         const struct rte_flow_item_gre gre_mask = {
6478                 .c_rsvd0_ver = RTE_BE16(0xB000),
6479                 .protocol = RTE_BE16(UINT16_MAX),
6480         };
6481         const struct rte_flow_item gre_item = {
6482                 .spec = &gre_spec,
6483                 .mask = &gre_mask,
6484                 .last = NULL,
6485         };
6486         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6487         if (!nvgre_v)
6488                 return;
6489         if (!nvgre_m)
6490                 nvgre_m = &rte_flow_item_nvgre_mask;
6491         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6492         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6493         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6494         memcpy(gre_key_m, tni_flow_id_m, size);
6495         for (i = 0; i < size; ++i)
6496                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6497 }
6498
6499 /**
6500  * Add VXLAN item to matcher and to the value.
6501  *
6502  * @param[in, out] matcher
6503  *   Flow matcher.
6504  * @param[in, out] key
6505  *   Flow matcher value.
6506  * @param[in] item
6507  *   Flow pattern to translate.
6508  * @param[in] inner
6509  *   Item is inner pattern.
6510  */
6511 static void
6512 flow_dv_translate_item_vxlan(void *matcher, void *key,
6513                              const struct rte_flow_item *item,
6514                              int inner)
6515 {
6516         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6517         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6518         void *headers_m;
6519         void *headers_v;
6520         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6521         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6522         char *vni_m;
6523         char *vni_v;
6524         uint16_t dport;
6525         int size;
6526         int i;
6527
6528         if (inner) {
6529                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6530                                          inner_headers);
6531                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6532         } else {
6533                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6534                                          outer_headers);
6535                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6536         }
6537         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6538                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6539         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6540                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6541                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6542         }
6543         if (!vxlan_v)
6544                 return;
6545         if (!vxlan_m)
6546                 vxlan_m = &rte_flow_item_vxlan_mask;
6547         size = sizeof(vxlan_m->vni);
6548         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6549         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6550         memcpy(vni_m, vxlan_m->vni, size);
6551         for (i = 0; i < size; ++i)
6552                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6553 }
6554
6555 /**
6556  * Add VXLAN-GPE item to matcher and to the value.
6557  *
6558  * @param[in, out] matcher
6559  *   Flow matcher.
6560  * @param[in, out] key
6561  *   Flow matcher value.
6562  * @param[in] item
6563  *   Flow pattern to translate.
6564  * @param[in] inner
6565  *   Item is inner pattern.
6566  */
6567
6568 static void
6569 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6570                                  const struct rte_flow_item *item, int inner)
6571 {
6572         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6573         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6574         void *headers_m;
6575         void *headers_v;
6576         void *misc_m =
6577                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6578         void *misc_v =
6579                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6580         char *vni_m;
6581         char *vni_v;
6582         uint16_t dport;
6583         int size;
6584         int i;
6585         uint8_t flags_m = 0xff;
6586         uint8_t flags_v = 0xc;
6587
6588         if (inner) {
6589                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6590                                          inner_headers);
6591                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6592         } else {
6593                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6594                                          outer_headers);
6595                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6596         }
6597         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6598                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6599         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6600                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6601                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6602         }
6603         if (!vxlan_v)
6604                 return;
6605         if (!vxlan_m)
6606                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6607         size = sizeof(vxlan_m->vni);
6608         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6609         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6610         memcpy(vni_m, vxlan_m->vni, size);
6611         for (i = 0; i < size; ++i)
6612                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6613         if (vxlan_m->flags) {
6614                 flags_m = vxlan_m->flags;
6615                 flags_v = vxlan_v->flags;
6616         }
6617         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6618         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6619         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6620                  vxlan_m->protocol);
6621         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6622                  vxlan_v->protocol);
6623 }
6624
6625 /**
6626  * Add Geneve item to matcher and to the value.
6627  *
6628  * @param[in, out] matcher
6629  *   Flow matcher.
6630  * @param[in, out] key
6631  *   Flow matcher value.
6632  * @param[in] item
6633  *   Flow pattern to translate.
6634  * @param[in] inner
6635  *   Item is inner pattern.
6636  */
6637
6638 static void
6639 flow_dv_translate_item_geneve(void *matcher, void *key,
6640                               const struct rte_flow_item *item, int inner)
6641 {
6642         const struct rte_flow_item_geneve *geneve_m = item->mask;
6643         const struct rte_flow_item_geneve *geneve_v = item->spec;
6644         void *headers_m;
6645         void *headers_v;
6646         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6647         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6648         uint16_t dport;
6649         uint16_t gbhdr_m;
6650         uint16_t gbhdr_v;
6651         char *vni_m;
6652         char *vni_v;
6653         size_t size, i;
6654
6655         if (inner) {
6656                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6657                                          inner_headers);
6658                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6659         } else {
6660                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6661                                          outer_headers);
6662                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6663         }
6664         dport = MLX5_UDP_PORT_GENEVE;
6665         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6666                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6667                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6668         }
6669         if (!geneve_v)
6670                 return;
6671         if (!geneve_m)
6672                 geneve_m = &rte_flow_item_geneve_mask;
6673         size = sizeof(geneve_m->vni);
6674         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6675         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6676         memcpy(vni_m, geneve_m->vni, size);
6677         for (i = 0; i < size; ++i)
6678                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6679         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6680                  rte_be_to_cpu_16(geneve_m->protocol));
6681         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6682                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6683         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6684         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6685         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6686                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6687         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6688                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6689         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6690                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6691         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6692                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6693                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6694 }
6695
6696 /**
6697  * Add MPLS item to matcher and to the value.
6698  *
6699  * @param[in, out] matcher
6700  *   Flow matcher.
6701  * @param[in, out] key
6702  *   Flow matcher value.
6703  * @param[in] item
6704  *   Flow pattern to translate.
6705  * @param[in] prev_layer
6706  *   The protocol layer indicated in previous item.
6707  * @param[in] inner
6708  *   Item is inner pattern.
6709  */
6710 static void
6711 flow_dv_translate_item_mpls(void *matcher, void *key,
6712                             const struct rte_flow_item *item,
6713                             uint64_t prev_layer,
6714                             int inner)
6715 {
6716         const uint32_t *in_mpls_m = item->mask;
6717         const uint32_t *in_mpls_v = item->spec;
6718         uint32_t *out_mpls_m = 0;
6719         uint32_t *out_mpls_v = 0;
6720         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6721         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6722         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6723                                      misc_parameters_2);
6724         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6725         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6726         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6727
6728         switch (prev_layer) {
6729         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6730                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6731                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6732                          MLX5_UDP_PORT_MPLS);
6733                 break;
6734         case MLX5_FLOW_LAYER_GRE:
6735                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6736                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6737                          RTE_ETHER_TYPE_MPLS);
6738                 break;
6739         default:
6740                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6741                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6742                          IPPROTO_MPLS);
6743                 break;
6744         }
6745         if (!in_mpls_v)
6746                 return;
6747         if (!in_mpls_m)
6748                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6749         switch (prev_layer) {
6750         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6751                 out_mpls_m =
6752                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6753                                                  outer_first_mpls_over_udp);
6754                 out_mpls_v =
6755                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6756                                                  outer_first_mpls_over_udp);
6757                 break;
6758         case MLX5_FLOW_LAYER_GRE:
6759                 out_mpls_m =
6760                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6761                                                  outer_first_mpls_over_gre);
6762                 out_mpls_v =
6763                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6764                                                  outer_first_mpls_over_gre);
6765                 break;
6766         default:
6767                 /* Inner MPLS not over GRE is not supported. */
6768                 if (!inner) {
6769                         out_mpls_m =
6770                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6771                                                          misc2_m,
6772                                                          outer_first_mpls);
6773                         out_mpls_v =
6774                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6775                                                          misc2_v,
6776                                                          outer_first_mpls);
6777                 }
6778                 break;
6779         }
6780         if (out_mpls_m && out_mpls_v) {
6781                 *out_mpls_m = *in_mpls_m;
6782                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6783         }
6784 }
6785
6786 /**
6787  * Add metadata register item to matcher
6788  *
6789  * @param[in, out] matcher
6790  *   Flow matcher.
6791  * @param[in, out] key
6792  *   Flow matcher value.
6793  * @param[in] reg_type
6794  *   Type of device metadata register
6795  * @param[in] value
6796  *   Register value
6797  * @param[in] mask
6798  *   Register mask
6799  */
6800 static void
6801 flow_dv_match_meta_reg(void *matcher, void *key,
6802                        enum modify_reg reg_type,
6803                        uint32_t data, uint32_t mask)
6804 {
6805         void *misc2_m =
6806                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6807         void *misc2_v =
6808                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6809         uint32_t temp;
6810
6811         data &= mask;
6812         switch (reg_type) {
6813         case REG_A:
6814                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6815                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6816                 break;
6817         case REG_B:
6818                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6819                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6820                 break;
6821         case REG_C_0:
6822                 /*
6823                  * The metadata register C0 field might be divided into
6824                  * source vport index and META item value, we should set
6825                  * this field according to specified mask, not as whole one.
6826                  */
6827                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6828                 temp |= mask;
6829                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6830                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6831                 temp &= ~mask;
6832                 temp |= data;
6833                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6834                 break;
6835         case REG_C_1:
6836                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6837                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6838                 break;
6839         case REG_C_2:
6840                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6841                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6842                 break;
6843         case REG_C_3:
6844                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6845                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6846                 break;
6847         case REG_C_4:
6848                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6849                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6850                 break;
6851         case REG_C_5:
6852                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6853                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6854                 break;
6855         case REG_C_6:
6856                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6857                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6858                 break;
6859         case REG_C_7:
6860                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6861                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6862                 break;
6863         default:
6864                 MLX5_ASSERT(false);
6865                 break;
6866         }
6867 }
6868
6869 /**
6870  * Add MARK item to matcher
6871  *
6872  * @param[in] dev
6873  *   The device to configure through.
6874  * @param[in, out] matcher
6875  *   Flow matcher.
6876  * @param[in, out] key
6877  *   Flow matcher value.
6878  * @param[in] item
6879  *   Flow pattern to translate.
6880  */
6881 static void
6882 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6883                             void *matcher, void *key,
6884                             const struct rte_flow_item *item)
6885 {
6886         struct mlx5_priv *priv = dev->data->dev_private;
6887         const struct rte_flow_item_mark *mark;
6888         uint32_t value;
6889         uint32_t mask;
6890
6891         mark = item->mask ? (const void *)item->mask :
6892                             &rte_flow_item_mark_mask;
6893         mask = mark->id & priv->sh->dv_mark_mask;
6894         mark = (const void *)item->spec;
6895         MLX5_ASSERT(mark);
6896         value = mark->id & priv->sh->dv_mark_mask & mask;
6897         if (mask) {
6898                 enum modify_reg reg;
6899
6900                 /* Get the metadata register index for the mark. */
6901                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6902                 MLX5_ASSERT(reg > 0);
6903                 if (reg == REG_C_0) {
6904                         struct mlx5_priv *priv = dev->data->dev_private;
6905                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6906                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6907
6908                         mask &= msk_c0;
6909                         mask <<= shl_c0;
6910                         value <<= shl_c0;
6911                 }
6912                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6913         }
6914 }
6915
6916 /**
6917  * Add META item to matcher
6918  *
6919  * @param[in] dev
6920  *   The devich to configure through.
6921  * @param[in, out] matcher
6922  *   Flow matcher.
6923  * @param[in, out] key
6924  *   Flow matcher value.
6925  * @param[in] attr
6926  *   Attributes of flow that includes this item.
6927  * @param[in] item
6928  *   Flow pattern to translate.
6929  */
6930 static void
6931 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6932                             void *matcher, void *key,
6933                             const struct rte_flow_attr *attr,
6934                             const struct rte_flow_item *item)
6935 {
6936         const struct rte_flow_item_meta *meta_m;
6937         const struct rte_flow_item_meta *meta_v;
6938
6939         meta_m = (const void *)item->mask;
6940         if (!meta_m)
6941                 meta_m = &rte_flow_item_meta_mask;
6942         meta_v = (const void *)item->spec;
6943         if (meta_v) {
6944                 int reg;
6945                 uint32_t value = meta_v->data;
6946                 uint32_t mask = meta_m->data;
6947
6948                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6949                 if (reg < 0)
6950                         return;
6951                 /*
6952                  * In datapath code there is no endianness
6953                  * coversions for perfromance reasons, all
6954                  * pattern conversions are done in rte_flow.
6955                  */
6956                 value = rte_cpu_to_be_32(value);
6957                 mask = rte_cpu_to_be_32(mask);
6958                 if (reg == REG_C_0) {
6959                         struct mlx5_priv *priv = dev->data->dev_private;
6960                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6961                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6962 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6963                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6964
6965                         value >>= shr_c0;
6966                         mask >>= shr_c0;
6967 #endif
6968                         value <<= shl_c0;
6969                         mask <<= shl_c0;
6970                         MLX5_ASSERT(msk_c0);
6971                         MLX5_ASSERT(!(~msk_c0 & mask));
6972                 }
6973                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6974         }
6975 }
6976
6977 /**
6978  * Add vport metadata Reg C0 item to matcher
6979  *
6980  * @param[in, out] matcher
6981  *   Flow matcher.
6982  * @param[in, out] key
6983  *   Flow matcher value.
6984  * @param[in] reg
6985  *   Flow pattern to translate.
6986  */
6987 static void
6988 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6989                                   uint32_t value, uint32_t mask)
6990 {
6991         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6992 }
6993
6994 /**
6995  * Add tag item to matcher
6996  *
6997  * @param[in] dev
6998  *   The devich to configure through.
6999  * @param[in, out] matcher
7000  *   Flow matcher.
7001  * @param[in, out] key
7002  *   Flow matcher value.
7003  * @param[in] item
7004  *   Flow pattern to translate.
7005  */
7006 static void
7007 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7008                                 void *matcher, void *key,
7009                                 const struct rte_flow_item *item)
7010 {
7011         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7012         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7013         uint32_t mask, value;
7014
7015         MLX5_ASSERT(tag_v);
7016         value = tag_v->data;
7017         mask = tag_m ? tag_m->data : UINT32_MAX;
7018         if (tag_v->id == REG_C_0) {
7019                 struct mlx5_priv *priv = dev->data->dev_private;
7020                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7021                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7022
7023                 mask &= msk_c0;
7024                 mask <<= shl_c0;
7025                 value <<= shl_c0;
7026         }
7027         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7028 }
7029
7030 /**
7031  * Add TAG item to matcher
7032  *
7033  * @param[in] dev
7034  *   The devich to configure through.
7035  * @param[in, out] matcher
7036  *   Flow matcher.
7037  * @param[in, out] key
7038  *   Flow matcher value.
7039  * @param[in] item
7040  *   Flow pattern to translate.
7041  */
7042 static void
7043 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7044                            void *matcher, void *key,
7045                            const struct rte_flow_item *item)
7046 {
7047         const struct rte_flow_item_tag *tag_v = item->spec;
7048         const struct rte_flow_item_tag *tag_m = item->mask;
7049         enum modify_reg reg;
7050
7051         MLX5_ASSERT(tag_v);
7052         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7053         /* Get the metadata register index for the tag. */
7054         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7055         MLX5_ASSERT(reg > 0);
7056         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7057 }
7058
7059 /**
7060  * Add source vport match to the specified matcher.
7061  *
7062  * @param[in, out] matcher
7063  *   Flow matcher.
7064  * @param[in, out] key
7065  *   Flow matcher value.
7066  * @param[in] port
7067  *   Source vport value to match
7068  * @param[in] mask
7069  *   Mask
7070  */
7071 static void
7072 flow_dv_translate_item_source_vport(void *matcher, void *key,
7073                                     int16_t port, uint16_t mask)
7074 {
7075         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7076         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7077
7078         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7079         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7080 }
7081
7082 /**
7083  * Translate port-id item to eswitch match on  port-id.
7084  *
7085  * @param[in] dev
7086  *   The devich to configure through.
7087  * @param[in, out] matcher
7088  *   Flow matcher.
7089  * @param[in, out] key
7090  *   Flow matcher value.
7091  * @param[in] item
7092  *   Flow pattern to translate.
7093  *
7094  * @return
7095  *   0 on success, a negative errno value otherwise.
7096  */
7097 static int
7098 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7099                                void *key, const struct rte_flow_item *item)
7100 {
7101         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7102         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7103         struct mlx5_priv *priv;
7104         uint16_t mask, id;
7105
7106         mask = pid_m ? pid_m->id : 0xffff;
7107         id = pid_v ? pid_v->id : dev->data->port_id;
7108         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7109         if (!priv)
7110                 return -rte_errno;
7111         /* Translate to vport field or to metadata, depending on mode. */
7112         if (priv->vport_meta_mask)
7113                 flow_dv_translate_item_meta_vport(matcher, key,
7114                                                   priv->vport_meta_tag,
7115                                                   priv->vport_meta_mask);
7116         else
7117                 flow_dv_translate_item_source_vport(matcher, key,
7118                                                     priv->vport_id, mask);
7119         return 0;
7120 }
7121
7122 /**
7123  * Add ICMP6 item to matcher and to the value.
7124  *
7125  * @param[in, out] matcher
7126  *   Flow matcher.
7127  * @param[in, out] key
7128  *   Flow matcher value.
7129  * @param[in] item
7130  *   Flow pattern to translate.
7131  * @param[in] inner
7132  *   Item is inner pattern.
7133  */
7134 static void
7135 flow_dv_translate_item_icmp6(void *matcher, void *key,
7136                               const struct rte_flow_item *item,
7137                               int inner)
7138 {
7139         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7140         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7141         void *headers_m;
7142         void *headers_v;
7143         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7144                                      misc_parameters_3);
7145         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7146         if (inner) {
7147                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7148                                          inner_headers);
7149                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7150         } else {
7151                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7152                                          outer_headers);
7153                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7154         }
7155         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7156         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7157         if (!icmp6_v)
7158                 return;
7159         if (!icmp6_m)
7160                 icmp6_m = &rte_flow_item_icmp6_mask;
7161         /*
7162          * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7163          * If only the protocol is specified, no need to match the frag.
7164          */
7165         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7166         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7167         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7168         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7169                  icmp6_v->type & icmp6_m->type);
7170         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7171         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7172                  icmp6_v->code & icmp6_m->code);
7173 }
7174
7175 /**
7176  * Add ICMP item to matcher and to the value.
7177  *
7178  * @param[in, out] matcher
7179  *   Flow matcher.
7180  * @param[in, out] key
7181  *   Flow matcher value.
7182  * @param[in] item
7183  *   Flow pattern to translate.
7184  * @param[in] inner
7185  *   Item is inner pattern.
7186  */
7187 static void
7188 flow_dv_translate_item_icmp(void *matcher, void *key,
7189                             const struct rte_flow_item *item,
7190                             int inner)
7191 {
7192         const struct rte_flow_item_icmp *icmp_m = item->mask;
7193         const struct rte_flow_item_icmp *icmp_v = item->spec;
7194         void *headers_m;
7195         void *headers_v;
7196         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7197                                      misc_parameters_3);
7198         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7199         if (inner) {
7200                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7201                                          inner_headers);
7202                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7203         } else {
7204                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7205                                          outer_headers);
7206                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7207         }
7208         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7209         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7210         if (!icmp_v)
7211                 return;
7212         if (!icmp_m)
7213                 icmp_m = &rte_flow_item_icmp_mask;
7214         /*
7215          * Force flow only to match the non-fragmented IPv4 ICMP packets.
7216          * If only the protocol is specified, no need to match the frag.
7217          */
7218         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7219         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7220         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7221                  icmp_m->hdr.icmp_type);
7222         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7223                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7224         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7225                  icmp_m->hdr.icmp_code);
7226         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7227                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7228 }
7229
7230 /**
7231  * Add GTP item to matcher and to the value.
7232  *
7233  * @param[in, out] matcher
7234  *   Flow matcher.
7235  * @param[in, out] key
7236  *   Flow matcher value.
7237  * @param[in] item
7238  *   Flow pattern to translate.
7239  * @param[in] inner
7240  *   Item is inner pattern.
7241  */
7242 static void
7243 flow_dv_translate_item_gtp(void *matcher, void *key,
7244                            const struct rte_flow_item *item, int inner)
7245 {
7246         const struct rte_flow_item_gtp *gtp_m = item->mask;
7247         const struct rte_flow_item_gtp *gtp_v = item->spec;
7248         void *headers_m;
7249         void *headers_v;
7250         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7251                                      misc_parameters_3);
7252         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7253         uint16_t dport = RTE_GTPU_UDP_PORT;
7254
7255         if (inner) {
7256                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7257                                          inner_headers);
7258                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7259         } else {
7260                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7261                                          outer_headers);
7262                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7263         }
7264         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7265                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7266                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7267         }
7268         if (!gtp_v)
7269                 return;
7270         if (!gtp_m)
7271                 gtp_m = &rte_flow_item_gtp_mask;
7272         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7273                  gtp_m->v_pt_rsv_flags);
7274         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7275                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7276         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7277         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7278                  gtp_v->msg_type & gtp_m->msg_type);
7279         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7280                  rte_be_to_cpu_32(gtp_m->teid));
7281         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7282                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7283 }
7284
7285 /**
7286  * Add eCPRI item to matcher and to the value.
7287  *
7288  * @param[in] dev
7289  *   The devich to configure through.
7290  * @param[in, out] matcher
7291  *   Flow matcher.
7292  * @param[in, out] key
7293  *   Flow matcher value.
7294  * @param[in] item
7295  *   Flow pattern to translate.
7296  * @param[in] samples
7297  *   Sample IDs to be used in the matching.
7298  */
7299 static void
7300 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7301                              void *key, const struct rte_flow_item *item)
7302 {
7303         struct mlx5_priv *priv = dev->data->dev_private;
7304         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7305         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7306         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7307                                      misc_parameters_4);
7308         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7309         uint32_t *samples;
7310         void *dw_m;
7311         void *dw_v;
7312
7313         if (!ecpri_v)
7314                 return;
7315         if (!ecpri_m)
7316                 ecpri_m = &rte_flow_item_ecpri_mask;
7317         /*
7318          * Maximal four DW samples are supported in a single matching now.
7319          * Two are used now for a eCPRI matching:
7320          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7321          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7322          *    if any.
7323          */
7324         if (!ecpri_m->hdr.common.u32)
7325                 return;
7326         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7327         /* Need to take the whole DW as the mask to fill the entry. */
7328         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7329                             prog_sample_field_value_0);
7330         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7331                             prog_sample_field_value_0);
7332         /* Already big endian (network order) in the header. */
7333         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7334         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7335         /* Sample#0, used for matching type, offset 0. */
7336         MLX5_SET(fte_match_set_misc4, misc4_m,
7337                  prog_sample_field_id_0, samples[0]);
7338         /* It makes no sense to set the sample ID in the mask field. */
7339         MLX5_SET(fte_match_set_misc4, misc4_v,
7340                  prog_sample_field_id_0, samples[0]);
7341         /*
7342          * Checking if message body part needs to be matched.
7343          * Some wildcard rules only matching type field should be supported.
7344          */
7345         if (ecpri_m->hdr.dummy[0]) {
7346                 switch (ecpri_v->hdr.common.type) {
7347                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7348                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7349                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7350                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7351                                             prog_sample_field_value_1);
7352                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7353                                             prog_sample_field_value_1);
7354                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7355                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7356                         /* Sample#1, to match message body, offset 4. */
7357                         MLX5_SET(fte_match_set_misc4, misc4_m,
7358                                  prog_sample_field_id_1, samples[1]);
7359                         MLX5_SET(fte_match_set_misc4, misc4_v,
7360                                  prog_sample_field_id_1, samples[1]);
7361                         break;
7362                 default:
7363                         /* Others, do not match any sample ID. */
7364                         break;
7365                 }
7366         }
7367 }
7368
7369 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7370
7371 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7372         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7373                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7374
7375 /**
7376  * Calculate flow matcher enable bitmap.
7377  *
7378  * @param match_criteria
7379  *   Pointer to flow matcher criteria.
7380  *
7381  * @return
7382  *   Bitmap of enabled fields.
7383  */
7384 static uint8_t
7385 flow_dv_matcher_enable(uint32_t *match_criteria)
7386 {
7387         uint8_t match_criteria_enable;
7388
7389         match_criteria_enable =
7390                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7391                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7392         match_criteria_enable |=
7393                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7394                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7395         match_criteria_enable |=
7396                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7397                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7398         match_criteria_enable |=
7399                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7400                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7401         match_criteria_enable |=
7402                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7403                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7404         match_criteria_enable |=
7405                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7406                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7407         return match_criteria_enable;
7408 }
7409
7410
7411 /**
7412  * Get a flow table.
7413  *
7414  * @param[in, out] dev
7415  *   Pointer to rte_eth_dev structure.
7416  * @param[in] table_id
7417  *   Table id to use.
7418  * @param[in] egress
7419  *   Direction of the table.
7420  * @param[in] transfer
7421  *   E-Switch or NIC flow.
7422  * @param[out] error
7423  *   pointer to error structure.
7424  *
7425  * @return
7426  *   Returns tables resource based on the index, NULL in case of failed.
7427  */
7428 static struct mlx5_flow_tbl_resource *
7429 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7430                          uint32_t table_id, uint8_t egress,
7431                          uint8_t transfer,
7432                          struct rte_flow_error *error)
7433 {
7434         struct mlx5_priv *priv = dev->data->dev_private;
7435         struct mlx5_dev_ctx_shared *sh = priv->sh;
7436         struct mlx5_flow_tbl_resource *tbl;
7437         union mlx5_flow_tbl_key table_key = {
7438                 {
7439                         .table_id = table_id,
7440                         .reserved = 0,
7441                         .domain = !!transfer,
7442                         .direction = !!egress,
7443                 }
7444         };
7445         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7446                                                          table_key.v64);
7447         struct mlx5_flow_tbl_data_entry *tbl_data;
7448         uint32_t idx = 0;
7449         int ret;
7450         void *domain;
7451
7452         if (pos) {
7453                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7454                                         entry);
7455                 tbl = &tbl_data->tbl;
7456                 rte_atomic32_inc(&tbl->refcnt);
7457                 return tbl;
7458         }
7459         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7460         if (!tbl_data) {
7461                 rte_flow_error_set(error, ENOMEM,
7462                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7463                                    NULL,
7464                                    "cannot allocate flow table data entry");
7465                 return NULL;
7466         }
7467         tbl_data->idx = idx;
7468         tbl = &tbl_data->tbl;
7469         pos = &tbl_data->entry;
7470         if (transfer)
7471                 domain = sh->fdb_domain;
7472         else if (egress)
7473                 domain = sh->tx_domain;
7474         else
7475                 domain = sh->rx_domain;
7476         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7477         if (ret) {
7478                 rte_flow_error_set(error, ENOMEM,
7479                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7480                                    NULL, "cannot create flow table object");
7481                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7482                 return NULL;
7483         }
7484         /*
7485          * No multi-threads now, but still better to initialize the reference
7486          * count before insert it into the hash list.
7487          */
7488         rte_atomic32_init(&tbl->refcnt);
7489         /* Jump action reference count is initialized here. */
7490         rte_atomic32_init(&tbl_data->jump.refcnt);
7491         pos->key = table_key.v64;
7492         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7493         if (ret < 0) {
7494                 rte_flow_error_set(error, -ret,
7495                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7496                                    "cannot insert flow table data entry");
7497                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7498                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7499         }
7500         rte_atomic32_inc(&tbl->refcnt);
7501         return tbl;
7502 }
7503
7504 /**
7505  * Release a flow table.
7506  *
7507  * @param[in] dev
7508  *   Pointer to rte_eth_dev structure.
7509  * @param[in] tbl
7510  *   Table resource to be released.
7511  *
7512  * @return
7513  *   Returns 0 if table was released, else return 1;
7514  */
7515 static int
7516 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7517                              struct mlx5_flow_tbl_resource *tbl)
7518 {
7519         struct mlx5_priv *priv = dev->data->dev_private;
7520         struct mlx5_dev_ctx_shared *sh = priv->sh;
7521         struct mlx5_flow_tbl_data_entry *tbl_data =
7522                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7523
7524         if (!tbl)
7525                 return 0;
7526         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7527                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7528
7529                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7530                 tbl->obj = NULL;
7531                 /* remove the entry from the hash list and free memory. */
7532                 mlx5_hlist_remove(sh->flow_tbls, pos);
7533                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7534                                 tbl_data->idx);
7535                 return 0;
7536         }
7537         return 1;
7538 }
7539
7540 /**
7541  * Register the flow matcher.
7542  *
7543  * @param[in, out] dev
7544  *   Pointer to rte_eth_dev structure.
7545  * @param[in, out] matcher
7546  *   Pointer to flow matcher.
7547  * @param[in, out] key
7548  *   Pointer to flow table key.
7549  * @parm[in, out] dev_flow
7550  *   Pointer to the dev_flow.
7551  * @param[out] error
7552  *   pointer to error structure.
7553  *
7554  * @return
7555  *   0 on success otherwise -errno and errno is set.
7556  */
7557 static int
7558 flow_dv_matcher_register(struct rte_eth_dev *dev,
7559                          struct mlx5_flow_dv_matcher *matcher,
7560                          union mlx5_flow_tbl_key *key,
7561                          struct mlx5_flow *dev_flow,
7562                          struct rte_flow_error *error)
7563 {
7564         struct mlx5_priv *priv = dev->data->dev_private;
7565         struct mlx5_dev_ctx_shared *sh = priv->sh;
7566         struct mlx5_flow_dv_matcher *cache_matcher;
7567         struct mlx5dv_flow_matcher_attr dv_attr = {
7568                 .type = IBV_FLOW_ATTR_NORMAL,
7569                 .match_mask = (void *)&matcher->mask,
7570         };
7571         struct mlx5_flow_tbl_resource *tbl;
7572         struct mlx5_flow_tbl_data_entry *tbl_data;
7573         int ret;
7574
7575         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7576                                        key->domain, error);
7577         if (!tbl)
7578                 return -rte_errno;      /* No need to refill the error info */
7579         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7580         /* Lookup from cache. */
7581         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7582                 if (matcher->crc == cache_matcher->crc &&
7583                     matcher->priority == cache_matcher->priority &&
7584                     !memcmp((const void *)matcher->mask.buf,
7585                             (const void *)cache_matcher->mask.buf,
7586                             cache_matcher->mask.size)) {
7587                         DRV_LOG(DEBUG,
7588                                 "%s group %u priority %hd use %s "
7589                                 "matcher %p: refcnt %d++",
7590                                 key->domain ? "FDB" : "NIC", key->table_id,
7591                                 cache_matcher->priority,
7592                                 key->direction ? "tx" : "rx",
7593                                 (void *)cache_matcher,
7594                                 rte_atomic32_read(&cache_matcher->refcnt));
7595                         rte_atomic32_inc(&cache_matcher->refcnt);
7596                         dev_flow->handle->dvh.matcher = cache_matcher;
7597                         /* old matcher should not make the table ref++. */
7598                         flow_dv_tbl_resource_release(dev, tbl);
7599                         return 0;
7600                 }
7601         }
7602         /* Register new matcher. */
7603         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
7604                                     SOCKET_ID_ANY);
7605         if (!cache_matcher) {
7606                 flow_dv_tbl_resource_release(dev, tbl);
7607                 return rte_flow_error_set(error, ENOMEM,
7608                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7609                                           "cannot allocate matcher memory");
7610         }
7611         *cache_matcher = *matcher;
7612         dv_attr.match_criteria_enable =
7613                 flow_dv_matcher_enable(cache_matcher->mask.buf);
7614         dv_attr.priority = matcher->priority;
7615         if (key->direction)
7616                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7617         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
7618                                                &cache_matcher->matcher_object);
7619         if (ret) {
7620                 mlx5_free(cache_matcher);
7621 #ifdef HAVE_MLX5DV_DR
7622                 flow_dv_tbl_resource_release(dev, tbl);
7623 #endif
7624                 return rte_flow_error_set(error, ENOMEM,
7625                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7626                                           NULL, "cannot create matcher");
7627         }
7628         /* Save the table information */
7629         cache_matcher->tbl = tbl;
7630         rte_atomic32_init(&cache_matcher->refcnt);
7631         /* only matcher ref++, table ref++ already done above in get API. */
7632         rte_atomic32_inc(&cache_matcher->refcnt);
7633         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7634         dev_flow->handle->dvh.matcher = cache_matcher;
7635         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7636                 key->domain ? "FDB" : "NIC", key->table_id,
7637                 cache_matcher->priority,
7638                 key->direction ? "tx" : "rx", (void *)cache_matcher,
7639                 rte_atomic32_read(&cache_matcher->refcnt));
7640         return 0;
7641 }
7642
7643 /**
7644  * Find existing tag resource or create and register a new one.
7645  *
7646  * @param dev[in, out]
7647  *   Pointer to rte_eth_dev structure.
7648  * @param[in, out] tag_be24
7649  *   Tag value in big endian then R-shift 8.
7650  * @parm[in, out] dev_flow
7651  *   Pointer to the dev_flow.
7652  * @param[out] error
7653  *   pointer to error structure.
7654  *
7655  * @return
7656  *   0 on success otherwise -errno and errno is set.
7657  */
7658 static int
7659 flow_dv_tag_resource_register
7660                         (struct rte_eth_dev *dev,
7661                          uint32_t tag_be24,
7662                          struct mlx5_flow *dev_flow,
7663                          struct rte_flow_error *error)
7664 {
7665         struct mlx5_priv *priv = dev->data->dev_private;
7666         struct mlx5_dev_ctx_shared *sh = priv->sh;
7667         struct mlx5_flow_dv_tag_resource *cache_resource;
7668         struct mlx5_hlist_entry *entry;
7669         int ret;
7670
7671         /* Lookup a matching resource from cache. */
7672         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7673         if (entry) {
7674                 cache_resource = container_of
7675                         (entry, struct mlx5_flow_dv_tag_resource, entry);
7676                 rte_atomic32_inc(&cache_resource->refcnt);
7677                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7678                 dev_flow->dv.tag_resource = cache_resource;
7679                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7680                         (void *)cache_resource,
7681                         rte_atomic32_read(&cache_resource->refcnt));
7682                 return 0;
7683         }
7684         /* Register new resource. */
7685         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7686                                        &dev_flow->handle->dvh.rix_tag);
7687         if (!cache_resource)
7688                 return rte_flow_error_set(error, ENOMEM,
7689                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7690                                           "cannot allocate resource memory");
7691         cache_resource->entry.key = (uint64_t)tag_be24;
7692         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
7693                                                   &cache_resource->action);
7694         if (ret) {
7695                 mlx5_free(cache_resource);
7696                 return rte_flow_error_set(error, ENOMEM,
7697                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7698                                           NULL, "cannot create action");
7699         }
7700         rte_atomic32_init(&cache_resource->refcnt);
7701         rte_atomic32_inc(&cache_resource->refcnt);
7702         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7703                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
7704                 mlx5_free(cache_resource);
7705                 return rte_flow_error_set(error, EEXIST,
7706                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7707                                           NULL, "cannot insert tag");
7708         }
7709         dev_flow->dv.tag_resource = cache_resource;
7710         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7711                 (void *)cache_resource,
7712                 rte_atomic32_read(&cache_resource->refcnt));
7713         return 0;
7714 }
7715
7716 /**
7717  * Release the tag.
7718  *
7719  * @param dev
7720  *   Pointer to Ethernet device.
7721  * @param tag_idx
7722  *   Tag index.
7723  *
7724  * @return
7725  *   1 while a reference on it exists, 0 when freed.
7726  */
7727 static int
7728 flow_dv_tag_release(struct rte_eth_dev *dev,
7729                     uint32_t tag_idx)
7730 {
7731         struct mlx5_priv *priv = dev->data->dev_private;
7732         struct mlx5_dev_ctx_shared *sh = priv->sh;
7733         struct mlx5_flow_dv_tag_resource *tag;
7734
7735         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7736         if (!tag)
7737                 return 0;
7738         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7739                 dev->data->port_id, (void *)tag,
7740                 rte_atomic32_read(&tag->refcnt));
7741         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7742                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
7743                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7744                 DRV_LOG(DEBUG, "port %u tag %p: removed",
7745                         dev->data->port_id, (void *)tag);
7746                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7747                 return 0;
7748         }
7749         return 1;
7750 }
7751
7752 /**
7753  * Translate port ID action to vport.
7754  *
7755  * @param[in] dev
7756  *   Pointer to rte_eth_dev structure.
7757  * @param[in] action
7758  *   Pointer to the port ID action.
7759  * @param[out] dst_port_id
7760  *   The target port ID.
7761  * @param[out] error
7762  *   Pointer to the error structure.
7763  *
7764  * @return
7765  *   0 on success, a negative errno value otherwise and rte_errno is set.
7766  */
7767 static int
7768 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7769                                  const struct rte_flow_action *action,
7770                                  uint32_t *dst_port_id,
7771                                  struct rte_flow_error *error)
7772 {
7773         uint32_t port;
7774         struct mlx5_priv *priv;
7775         const struct rte_flow_action_port_id *conf =
7776                         (const struct rte_flow_action_port_id *)action->conf;
7777
7778         port = conf->original ? dev->data->port_id : conf->id;
7779         priv = mlx5_port_to_eswitch_info(port, false);
7780         if (!priv)
7781                 return rte_flow_error_set(error, -rte_errno,
7782                                           RTE_FLOW_ERROR_TYPE_ACTION,
7783                                           NULL,
7784                                           "No eswitch info was found for port");
7785 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7786         /*
7787          * This parameter is transferred to
7788          * mlx5dv_dr_action_create_dest_ib_port().
7789          */
7790         *dst_port_id = priv->dev_port;
7791 #else
7792         /*
7793          * Legacy mode, no LAG configurations is supported.
7794          * This parameter is transferred to
7795          * mlx5dv_dr_action_create_dest_vport().
7796          */
7797         *dst_port_id = priv->vport_id;
7798 #endif
7799         return 0;
7800 }
7801
7802 /**
7803  * Create a counter with aging configuration.
7804  *
7805  * @param[in] dev
7806  *   Pointer to rte_eth_dev structure.
7807  * @param[out] count
7808  *   Pointer to the counter action configuration.
7809  * @param[in] age
7810  *   Pointer to the aging action configuration.
7811  *
7812  * @return
7813  *   Index to flow counter on success, 0 otherwise.
7814  */
7815 static uint32_t
7816 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7817                                 struct mlx5_flow *dev_flow,
7818                                 const struct rte_flow_action_count *count,
7819                                 const struct rte_flow_action_age *age)
7820 {
7821         uint32_t counter;
7822         struct mlx5_age_param *age_param;
7823
7824         counter = flow_dv_counter_alloc(dev,
7825                                 count ? count->shared : 0,
7826                                 count ? count->id : 0,
7827                                 dev_flow->dv.group, !!age);
7828         if (!counter || age == NULL)
7829                 return counter;
7830         age_param  = flow_dv_counter_idx_get_age(dev, counter);
7831         /*
7832          * The counter age accuracy may have a bit delay. Have 3/4
7833          * second bias on the timeount in order to let it age in time.
7834          */
7835         age_param->context = age->context ? age->context :
7836                 (void *)(uintptr_t)(dev_flow->flow_idx);
7837         /*
7838          * The counter age accuracy may have a bit delay. Have 3/4
7839          * second bias on the timeount in order to let it age in time.
7840          */
7841         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7842         /* Set expire time in unit of 0.1 sec. */
7843         age_param->port_id = dev->data->port_id;
7844         age_param->expire = age_param->timeout +
7845                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
7846         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7847         return counter;
7848 }
7849 /**
7850  * Add Tx queue matcher
7851  *
7852  * @param[in] dev
7853  *   Pointer to the dev struct.
7854  * @param[in, out] matcher
7855  *   Flow matcher.
7856  * @param[in, out] key
7857  *   Flow matcher value.
7858  * @param[in] item
7859  *   Flow pattern to translate.
7860  * @param[in] inner
7861  *   Item is inner pattern.
7862  */
7863 static void
7864 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7865                                 void *matcher, void *key,
7866                                 const struct rte_flow_item *item)
7867 {
7868         const struct mlx5_rte_flow_item_tx_queue *queue_m;
7869         const struct mlx5_rte_flow_item_tx_queue *queue_v;
7870         void *misc_m =
7871                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7872         void *misc_v =
7873                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7874         struct mlx5_txq_ctrl *txq;
7875         uint32_t queue;
7876
7877
7878         queue_m = (const void *)item->mask;
7879         if (!queue_m)
7880                 return;
7881         queue_v = (const void *)item->spec;
7882         if (!queue_v)
7883                 return;
7884         txq = mlx5_txq_get(dev, queue_v->queue);
7885         if (!txq)
7886                 return;
7887         queue = txq->obj->sq->id;
7888         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7889         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7890                  queue & queue_m->queue);
7891         mlx5_txq_release(dev, queue_v->queue);
7892 }
7893
7894 /**
7895  * Set the hash fields according to the @p flow information.
7896  *
7897  * @param[in] dev_flow
7898  *   Pointer to the mlx5_flow.
7899  * @param[in] rss_desc
7900  *   Pointer to the mlx5_flow_rss_desc.
7901  */
7902 static void
7903 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7904                        struct mlx5_flow_rss_desc *rss_desc)
7905 {
7906         uint64_t items = dev_flow->handle->layers;
7907         int rss_inner = 0;
7908         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7909
7910         dev_flow->hash_fields = 0;
7911 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7912         if (rss_desc->level >= 2) {
7913                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7914                 rss_inner = 1;
7915         }
7916 #endif
7917         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7918             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7919                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7920                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7921                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7922                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7923                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7924                         else
7925                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7926                 }
7927         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7928                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7929                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7930                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7931                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7932                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7933                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7934                         else
7935                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7936                 }
7937         }
7938         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7939             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7940                 if (rss_types & ETH_RSS_UDP) {
7941                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7942                                 dev_flow->hash_fields |=
7943                                                 IBV_RX_HASH_SRC_PORT_UDP;
7944                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7945                                 dev_flow->hash_fields |=
7946                                                 IBV_RX_HASH_DST_PORT_UDP;
7947                         else
7948                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7949                 }
7950         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7951                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7952                 if (rss_types & ETH_RSS_TCP) {
7953                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7954                                 dev_flow->hash_fields |=
7955                                                 IBV_RX_HASH_SRC_PORT_TCP;
7956                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7957                                 dev_flow->hash_fields |=
7958                                                 IBV_RX_HASH_DST_PORT_TCP;
7959                         else
7960                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7961                 }
7962         }
7963 }
7964
7965 /**
7966  * Fill the flow with DV spec, lock free
7967  * (mutex should be acquired by caller).
7968  *
7969  * @param[in] dev
7970  *   Pointer to rte_eth_dev structure.
7971  * @param[in, out] dev_flow
7972  *   Pointer to the sub flow.
7973  * @param[in] attr
7974  *   Pointer to the flow attributes.
7975  * @param[in] items
7976  *   Pointer to the list of items.
7977  * @param[in] actions
7978  *   Pointer to the list of actions.
7979  * @param[out] error
7980  *   Pointer to the error structure.
7981  *
7982  * @return
7983  *   0 on success, a negative errno value otherwise and rte_errno is set.
7984  */
7985 static int
7986 __flow_dv_translate(struct rte_eth_dev *dev,
7987                     struct mlx5_flow *dev_flow,
7988                     const struct rte_flow_attr *attr,
7989                     const struct rte_flow_item items[],
7990                     const struct rte_flow_action actions[],
7991                     struct rte_flow_error *error)
7992 {
7993         struct mlx5_priv *priv = dev->data->dev_private;
7994         struct mlx5_dev_config *dev_conf = &priv->config;
7995         struct rte_flow *flow = dev_flow->flow;
7996         struct mlx5_flow_handle *handle = dev_flow->handle;
7997         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7998                                               priv->rss_desc)
7999                                               [!!priv->flow_nested_idx];
8000         uint64_t item_flags = 0;
8001         uint64_t last_item = 0;
8002         uint64_t action_flags = 0;
8003         uint64_t priority = attr->priority;
8004         struct mlx5_flow_dv_matcher matcher = {
8005                 .mask = {
8006                         .size = sizeof(matcher.mask.buf) -
8007                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
8008                 },
8009         };
8010         int actions_n = 0;
8011         bool actions_end = false;
8012         union {
8013                 struct mlx5_flow_dv_modify_hdr_resource res;
8014                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
8015                             sizeof(struct mlx5_modification_cmd) *
8016                             (MLX5_MAX_MODIFY_NUM + 1)];
8017         } mhdr_dummy;
8018         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
8019         const struct rte_flow_action_count *count = NULL;
8020         const struct rte_flow_action_age *age = NULL;
8021         union flow_dv_attr flow_attr = { .attr = 0 };
8022         uint32_t tag_be;
8023         union mlx5_flow_tbl_key tbl_key;
8024         uint32_t modify_action_position = UINT32_MAX;
8025         void *match_mask = matcher.mask.buf;
8026         void *match_value = dev_flow->dv.value.buf;
8027         uint8_t next_protocol = 0xff;
8028         struct rte_vlan_hdr vlan = { 0 };
8029         uint32_t table;
8030         int ret = 0;
8031
8032         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
8033                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
8034         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
8035                                        !!priv->fdb_def_rule, &table, error);
8036         if (ret)
8037                 return ret;
8038         dev_flow->dv.group = table;
8039         if (attr->transfer)
8040                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
8041         if (priority == MLX5_FLOW_PRIO_RSVD)
8042                 priority = dev_conf->flow_prio - 1;
8043         /* number of actions must be set to 0 in case of dirty stack. */
8044         mhdr_res->actions_num = 0;
8045         for (; !actions_end ; actions++) {
8046                 const struct rte_flow_action_queue *queue;
8047                 const struct rte_flow_action_rss *rss;
8048                 const struct rte_flow_action *action = actions;
8049                 const uint8_t *rss_key;
8050                 const struct rte_flow_action_jump *jump_data;
8051                 const struct rte_flow_action_meter *mtr;
8052                 struct mlx5_flow_tbl_resource *tbl;
8053                 uint32_t port_id = 0;
8054                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
8055                 int action_type = actions->type;
8056                 const struct rte_flow_action *found_action = NULL;
8057                 struct mlx5_flow_meter *fm = NULL;
8058
8059                 if (!mlx5_flow_os_action_supported(action_type))
8060                         return rte_flow_error_set(error, ENOTSUP,
8061                                                   RTE_FLOW_ERROR_TYPE_ACTION,
8062                                                   actions,
8063                                                   "action not supported");
8064                 switch (action_type) {
8065                 case RTE_FLOW_ACTION_TYPE_VOID:
8066                         break;
8067                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8068                         if (flow_dv_translate_action_port_id(dev, action,
8069                                                              &port_id, error))
8070                                 return -rte_errno;
8071                         port_id_resource.port_id = port_id;
8072                         MLX5_ASSERT(!handle->rix_port_id_action);
8073                         if (flow_dv_port_id_action_resource_register
8074                             (dev, &port_id_resource, dev_flow, error))
8075                                 return -rte_errno;
8076                         dev_flow->dv.actions[actions_n++] =
8077                                         dev_flow->dv.port_id_action->action;
8078                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8079                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
8080                         break;
8081                 case RTE_FLOW_ACTION_TYPE_FLAG:
8082                         action_flags |= MLX5_FLOW_ACTION_FLAG;
8083                         dev_flow->handle->mark = 1;
8084                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8085                                 struct rte_flow_action_mark mark = {
8086                                         .id = MLX5_FLOW_MARK_DEFAULT,
8087                                 };
8088
8089                                 if (flow_dv_convert_action_mark(dev, &mark,
8090                                                                 mhdr_res,
8091                                                                 error))
8092                                         return -rte_errno;
8093                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8094                                 break;
8095                         }
8096                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
8097                         /*
8098                          * Only one FLAG or MARK is supported per device flow
8099                          * right now. So the pointer to the tag resource must be
8100                          * zero before the register process.
8101                          */
8102                         MLX5_ASSERT(!handle->dvh.rix_tag);
8103                         if (flow_dv_tag_resource_register(dev, tag_be,
8104                                                           dev_flow, error))
8105                                 return -rte_errno;
8106                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8107                         dev_flow->dv.actions[actions_n++] =
8108                                         dev_flow->dv.tag_resource->action;
8109                         break;
8110                 case RTE_FLOW_ACTION_TYPE_MARK:
8111                         action_flags |= MLX5_FLOW_ACTION_MARK;
8112                         dev_flow->handle->mark = 1;
8113                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8114                                 const struct rte_flow_action_mark *mark =
8115                                         (const struct rte_flow_action_mark *)
8116                                                 actions->conf;
8117
8118                                 if (flow_dv_convert_action_mark(dev, mark,
8119                                                                 mhdr_res,
8120                                                                 error))
8121                                         return -rte_errno;
8122                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8123                                 break;
8124                         }
8125                         /* Fall-through */
8126                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
8127                         /* Legacy (non-extensive) MARK action. */
8128                         tag_be = mlx5_flow_mark_set
8129                               (((const struct rte_flow_action_mark *)
8130                                (actions->conf))->id);
8131                         MLX5_ASSERT(!handle->dvh.rix_tag);
8132                         if (flow_dv_tag_resource_register(dev, tag_be,
8133                                                           dev_flow, error))
8134                                 return -rte_errno;
8135                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8136                         dev_flow->dv.actions[actions_n++] =
8137                                         dev_flow->dv.tag_resource->action;
8138                         break;
8139                 case RTE_FLOW_ACTION_TYPE_SET_META:
8140                         if (flow_dv_convert_action_set_meta
8141                                 (dev, mhdr_res, attr,
8142                                  (const struct rte_flow_action_set_meta *)
8143                                   actions->conf, error))
8144                                 return -rte_errno;
8145                         action_flags |= MLX5_FLOW_ACTION_SET_META;
8146                         break;
8147                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
8148                         if (flow_dv_convert_action_set_tag
8149                                 (dev, mhdr_res,
8150                                  (const struct rte_flow_action_set_tag *)
8151                                   actions->conf, error))
8152                                 return -rte_errno;
8153                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8154                         break;
8155                 case RTE_FLOW_ACTION_TYPE_DROP:
8156                         action_flags |= MLX5_FLOW_ACTION_DROP;
8157                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8158                         break;
8159                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8160                         queue = actions->conf;
8161                         rss_desc->queue_num = 1;
8162                         rss_desc->queue[0] = queue->index;
8163                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8164                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8165                         break;
8166                 case RTE_FLOW_ACTION_TYPE_RSS:
8167                         rss = actions->conf;
8168                         memcpy(rss_desc->queue, rss->queue,
8169                                rss->queue_num * sizeof(uint16_t));
8170                         rss_desc->queue_num = rss->queue_num;
8171                         /* NULL RSS key indicates default RSS key. */
8172                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
8173                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8174                         /*
8175                          * rss->level and rss.types should be set in advance
8176                          * when expanding items for RSS.
8177                          */
8178                         action_flags |= MLX5_FLOW_ACTION_RSS;
8179                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8180                         break;
8181                 case RTE_FLOW_ACTION_TYPE_AGE:
8182                 case RTE_FLOW_ACTION_TYPE_COUNT:
8183                         if (!dev_conf->devx) {
8184                                 return rte_flow_error_set
8185                                               (error, ENOTSUP,
8186                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8187                                                NULL,
8188                                                "count action not supported");
8189                         }
8190                         /* Save information first, will apply later. */
8191                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8192                                 count = action->conf;
8193                         else
8194                                 age = action->conf;
8195                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8196                         break;
8197                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8198                         dev_flow->dv.actions[actions_n++] =
8199                                                 priv->sh->pop_vlan_action;
8200                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8201                         break;
8202                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8203                         if (!(action_flags &
8204                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8205                                 flow_dev_get_vlan_info_from_items(items, &vlan);
8206                         vlan.eth_proto = rte_be_to_cpu_16
8207                              ((((const struct rte_flow_action_of_push_vlan *)
8208                                                    actions->conf)->ethertype));
8209                         found_action = mlx5_flow_find_action
8210                                         (actions + 1,
8211                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8212                         if (found_action)
8213                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8214                         found_action = mlx5_flow_find_action
8215                                         (actions + 1,
8216                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8217                         if (found_action)
8218                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8219                         if (flow_dv_create_action_push_vlan
8220                                             (dev, attr, &vlan, dev_flow, error))
8221                                 return -rte_errno;
8222                         dev_flow->dv.actions[actions_n++] =
8223                                         dev_flow->dv.push_vlan_res->action;
8224                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8225                         break;
8226                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8227                         /* of_vlan_push action handled this action */
8228                         MLX5_ASSERT(action_flags &
8229                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8230                         break;
8231                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8232                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8233                                 break;
8234                         flow_dev_get_vlan_info_from_items(items, &vlan);
8235                         mlx5_update_vlan_vid_pcp(actions, &vlan);
8236                         /* If no VLAN push - this is a modify header action */
8237                         if (flow_dv_convert_action_modify_vlan_vid
8238                                                 (mhdr_res, actions, error))
8239                                 return -rte_errno;
8240                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8241                         break;
8242                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8243                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8244                         if (flow_dv_create_action_l2_encap(dev, actions,
8245                                                            dev_flow,
8246                                                            attr->transfer,
8247                                                            error))
8248                                 return -rte_errno;
8249                         dev_flow->dv.actions[actions_n++] =
8250                                         dev_flow->dv.encap_decap->action;
8251                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8252                         break;
8253                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8254                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8255                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
8256                                                            attr->transfer,
8257                                                            error))
8258                                 return -rte_errno;
8259                         dev_flow->dv.actions[actions_n++] =
8260                                         dev_flow->dv.encap_decap->action;
8261                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8262                         break;
8263                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8264                         /* Handle encap with preceding decap. */
8265                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8266                                 if (flow_dv_create_action_raw_encap
8267                                         (dev, actions, dev_flow, attr, error))
8268                                         return -rte_errno;
8269                                 dev_flow->dv.actions[actions_n++] =
8270                                         dev_flow->dv.encap_decap->action;
8271                         } else {
8272                                 /* Handle encap without preceding decap. */
8273                                 if (flow_dv_create_action_l2_encap
8274                                     (dev, actions, dev_flow, attr->transfer,
8275                                      error))
8276                                         return -rte_errno;
8277                                 dev_flow->dv.actions[actions_n++] =
8278                                         dev_flow->dv.encap_decap->action;
8279                         }
8280                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8281                         break;
8282                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8283                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8284                                 ;
8285                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8286                                 if (flow_dv_create_action_l2_decap
8287                                     (dev, dev_flow, attr->transfer, error))
8288                                         return -rte_errno;
8289                                 dev_flow->dv.actions[actions_n++] =
8290                                         dev_flow->dv.encap_decap->action;
8291                         }
8292                         /* If decap is followed by encap, handle it at encap. */
8293                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8294                         break;
8295                 case RTE_FLOW_ACTION_TYPE_JUMP:
8296                         jump_data = action->conf;
8297                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8298                                                        jump_data->group,
8299                                                        !!priv->fdb_def_rule,
8300                                                        &table, error);
8301                         if (ret)
8302                                 return ret;
8303                         tbl = flow_dv_tbl_resource_get(dev, table,
8304                                                        attr->egress,
8305                                                        attr->transfer, error);
8306                         if (!tbl)
8307                                 return rte_flow_error_set
8308                                                 (error, errno,
8309                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8310                                                  NULL,
8311                                                  "cannot create jump action.");
8312                         if (flow_dv_jump_tbl_resource_register
8313                             (dev, tbl, dev_flow, error)) {
8314                                 flow_dv_tbl_resource_release(dev, tbl);
8315                                 return rte_flow_error_set
8316                                                 (error, errno,
8317                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8318                                                  NULL,
8319                                                  "cannot create jump action.");
8320                         }
8321                         dev_flow->dv.actions[actions_n++] =
8322                                         dev_flow->dv.jump->action;
8323                         action_flags |= MLX5_FLOW_ACTION_JUMP;
8324                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8325                         break;
8326                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8327                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8328                         if (flow_dv_convert_action_modify_mac
8329                                         (mhdr_res, actions, error))
8330                                 return -rte_errno;
8331                         action_flags |= actions->type ==
8332                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8333                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
8334                                         MLX5_FLOW_ACTION_SET_MAC_DST;
8335                         break;
8336                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8337                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8338                         if (flow_dv_convert_action_modify_ipv4
8339                                         (mhdr_res, actions, error))
8340                                 return -rte_errno;
8341                         action_flags |= actions->type ==
8342                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8343                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
8344                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
8345                         break;
8346                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8347                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8348                         if (flow_dv_convert_action_modify_ipv6
8349                                         (mhdr_res, actions, error))
8350                                 return -rte_errno;
8351                         action_flags |= actions->type ==
8352                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8353                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
8354                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
8355                         break;
8356                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8357                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8358                         if (flow_dv_convert_action_modify_tp
8359                                         (mhdr_res, actions, items,
8360                                          &flow_attr, dev_flow, !!(action_flags &
8361                                          MLX5_FLOW_ACTION_DECAP), error))
8362                                 return -rte_errno;
8363                         action_flags |= actions->type ==
8364                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8365                                         MLX5_FLOW_ACTION_SET_TP_SRC :
8366                                         MLX5_FLOW_ACTION_SET_TP_DST;
8367                         break;
8368                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8369                         if (flow_dv_convert_action_modify_dec_ttl
8370                                         (mhdr_res, items, &flow_attr, dev_flow,
8371                                          !!(action_flags &
8372                                          MLX5_FLOW_ACTION_DECAP), error))
8373                                 return -rte_errno;
8374                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8375                         break;
8376                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8377                         if (flow_dv_convert_action_modify_ttl
8378                                         (mhdr_res, actions, items, &flow_attr,
8379                                          dev_flow, !!(action_flags &
8380                                          MLX5_FLOW_ACTION_DECAP), error))
8381                                 return -rte_errno;
8382                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8383                         break;
8384                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8385                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8386                         if (flow_dv_convert_action_modify_tcp_seq
8387                                         (mhdr_res, actions, error))
8388                                 return -rte_errno;
8389                         action_flags |= actions->type ==
8390                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8391                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
8392                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8393                         break;
8394
8395                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8396                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8397                         if (flow_dv_convert_action_modify_tcp_ack
8398                                         (mhdr_res, actions, error))
8399                                 return -rte_errno;
8400                         action_flags |= actions->type ==
8401                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8402                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
8403                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
8404                         break;
8405                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8406                         if (flow_dv_convert_action_set_reg
8407                                         (mhdr_res, actions, error))
8408                                 return -rte_errno;
8409                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8410                         break;
8411                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8412                         if (flow_dv_convert_action_copy_mreg
8413                                         (dev, mhdr_res, actions, error))
8414                                 return -rte_errno;
8415                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8416                         break;
8417                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8418                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8419                         dev_flow->handle->fate_action =
8420                                         MLX5_FLOW_FATE_DEFAULT_MISS;
8421                         break;
8422                 case RTE_FLOW_ACTION_TYPE_METER:
8423                         mtr = actions->conf;
8424                         if (!flow->meter) {
8425                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8426                                                             attr, error);
8427                                 if (!fm)
8428                                         return rte_flow_error_set(error,
8429                                                 rte_errno,
8430                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8431                                                 NULL,
8432                                                 "meter not found "
8433                                                 "or invalid parameters");
8434                                 flow->meter = fm->idx;
8435                         }
8436                         /* Set the meter action. */
8437                         if (!fm) {
8438                                 fm = mlx5_ipool_get(priv->sh->ipool
8439                                                 [MLX5_IPOOL_MTR], flow->meter);
8440                                 if (!fm)
8441                                         return rte_flow_error_set(error,
8442                                                 rte_errno,
8443                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8444                                                 NULL,
8445                                                 "meter not found "
8446                                                 "or invalid parameters");
8447                         }
8448                         dev_flow->dv.actions[actions_n++] =
8449                                 fm->mfts->meter_action;
8450                         action_flags |= MLX5_FLOW_ACTION_METER;
8451                         break;
8452                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8453                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8454                                                               actions, error))
8455                                 return -rte_errno;
8456                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8457                         break;
8458                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8459                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8460                                                               actions, error))
8461                                 return -rte_errno;
8462                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8463                         break;
8464                 case RTE_FLOW_ACTION_TYPE_END:
8465                         actions_end = true;
8466                         if (mhdr_res->actions_num) {
8467                                 /* create modify action if needed. */
8468                                 if (flow_dv_modify_hdr_resource_register
8469                                         (dev, mhdr_res, dev_flow, error))
8470                                         return -rte_errno;
8471                                 dev_flow->dv.actions[modify_action_position] =
8472                                         handle->dvh.modify_hdr->action;
8473                         }
8474                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8475                                 flow->counter =
8476                                         flow_dv_translate_create_counter(dev,
8477                                                 dev_flow, count, age);
8478
8479                                 if (!flow->counter)
8480                                         return rte_flow_error_set
8481                                                 (error, rte_errno,
8482                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8483                                                 NULL,
8484                                                 "cannot create counter"
8485                                                 " object.");
8486                                 dev_flow->dv.actions[actions_n++] =
8487                                           (flow_dv_counter_get_by_idx(dev,
8488                                           flow->counter, NULL))->action;
8489                         }
8490                         break;
8491                 default:
8492                         break;
8493                 }
8494                 if (mhdr_res->actions_num &&
8495                     modify_action_position == UINT32_MAX)
8496                         modify_action_position = actions_n++;
8497         }
8498         dev_flow->dv.actions_n = actions_n;
8499         dev_flow->act_flags = action_flags;
8500         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8501                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8502                 int item_type = items->type;
8503
8504                 if (!mlx5_flow_os_item_supported(item_type))
8505                         return rte_flow_error_set(error, ENOTSUP,
8506                                                   RTE_FLOW_ERROR_TYPE_ITEM,
8507                                                   NULL, "item not supported");
8508                 switch (item_type) {
8509                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8510                         flow_dv_translate_item_port_id(dev, match_mask,
8511                                                        match_value, items);
8512                         last_item = MLX5_FLOW_ITEM_PORT_ID;
8513                         break;
8514                 case RTE_FLOW_ITEM_TYPE_ETH:
8515                         flow_dv_translate_item_eth(match_mask, match_value,
8516                                                    items, tunnel,
8517                                                    dev_flow->dv.group);
8518                         matcher.priority = action_flags &
8519                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
8520                                         !dev_flow->external ?
8521                                         MLX5_PRIORITY_MAP_L3 :
8522                                         MLX5_PRIORITY_MAP_L2;
8523                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8524                                              MLX5_FLOW_LAYER_OUTER_L2;
8525                         break;
8526                 case RTE_FLOW_ITEM_TYPE_VLAN:
8527                         flow_dv_translate_item_vlan(dev_flow,
8528                                                     match_mask, match_value,
8529                                                     items, tunnel,
8530                                                     dev_flow->dv.group);
8531                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8532                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8533                                               MLX5_FLOW_LAYER_INNER_VLAN) :
8534                                              (MLX5_FLOW_LAYER_OUTER_L2 |
8535                                               MLX5_FLOW_LAYER_OUTER_VLAN);
8536                         break;
8537                 case RTE_FLOW_ITEM_TYPE_IPV4:
8538                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8539                                                   &item_flags, &tunnel);
8540                         flow_dv_translate_item_ipv4(match_mask, match_value,
8541                                                     items, item_flags, tunnel,
8542                                                     dev_flow->dv.group);
8543                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8544                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8545                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8546                         if (items->mask != NULL &&
8547                             ((const struct rte_flow_item_ipv4 *)
8548                              items->mask)->hdr.next_proto_id) {
8549                                 next_protocol =
8550                                         ((const struct rte_flow_item_ipv4 *)
8551                                          (items->spec))->hdr.next_proto_id;
8552                                 next_protocol &=
8553                                         ((const struct rte_flow_item_ipv4 *)
8554                                          (items->mask))->hdr.next_proto_id;
8555                         } else {
8556                                 /* Reset for inner layer. */
8557                                 next_protocol = 0xff;
8558                         }
8559                         break;
8560                 case RTE_FLOW_ITEM_TYPE_IPV6:
8561                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8562                                                   &item_flags, &tunnel);
8563                         flow_dv_translate_item_ipv6(match_mask, match_value,
8564                                                     items, item_flags, tunnel,
8565                                                     dev_flow->dv.group);
8566                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8567                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8568                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8569                         if (items->mask != NULL &&
8570                             ((const struct rte_flow_item_ipv6 *)
8571                              items->mask)->hdr.proto) {
8572                                 next_protocol =
8573                                         ((const struct rte_flow_item_ipv6 *)
8574                                          items->spec)->hdr.proto;
8575                                 next_protocol &=
8576                                         ((const struct rte_flow_item_ipv6 *)
8577                                          items->mask)->hdr.proto;
8578                         } else {
8579                                 /* Reset for inner layer. */
8580                                 next_protocol = 0xff;
8581                         }
8582                         break;
8583                 case RTE_FLOW_ITEM_TYPE_TCP:
8584                         flow_dv_translate_item_tcp(match_mask, match_value,
8585                                                    items, tunnel);
8586                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8587                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8588                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
8589                         break;
8590                 case RTE_FLOW_ITEM_TYPE_UDP:
8591                         flow_dv_translate_item_udp(match_mask, match_value,
8592                                                    items, tunnel);
8593                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8594                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8595                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
8596                         break;
8597                 case RTE_FLOW_ITEM_TYPE_GRE:
8598                         flow_dv_translate_item_gre(match_mask, match_value,
8599                                                    items, tunnel);
8600                         matcher.priority = rss_desc->level >= 2 ?
8601                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8602                         last_item = MLX5_FLOW_LAYER_GRE;
8603                         break;
8604                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8605                         flow_dv_translate_item_gre_key(match_mask,
8606                                                        match_value, items);
8607                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
8608                         break;
8609                 case RTE_FLOW_ITEM_TYPE_NVGRE:
8610                         flow_dv_translate_item_nvgre(match_mask, match_value,
8611                                                      items, tunnel);
8612                         matcher.priority = rss_desc->level >= 2 ?
8613                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8614                         last_item = MLX5_FLOW_LAYER_GRE;
8615                         break;
8616                 case RTE_FLOW_ITEM_TYPE_VXLAN:
8617                         flow_dv_translate_item_vxlan(match_mask, match_value,
8618                                                      items, tunnel);
8619                         matcher.priority = rss_desc->level >= 2 ?
8620                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8621                         last_item = MLX5_FLOW_LAYER_VXLAN;
8622                         break;
8623                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8624                         flow_dv_translate_item_vxlan_gpe(match_mask,
8625                                                          match_value, items,
8626                                                          tunnel);
8627                         matcher.priority = rss_desc->level >= 2 ?
8628                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8629                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8630                         break;
8631                 case RTE_FLOW_ITEM_TYPE_GENEVE:
8632                         flow_dv_translate_item_geneve(match_mask, match_value,
8633                                                       items, tunnel);
8634                         matcher.priority = rss_desc->level >= 2 ?
8635                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8636                         last_item = MLX5_FLOW_LAYER_GENEVE;
8637                         break;
8638                 case RTE_FLOW_ITEM_TYPE_MPLS:
8639                         flow_dv_translate_item_mpls(match_mask, match_value,
8640                                                     items, last_item, tunnel);
8641                         matcher.priority = rss_desc->level >= 2 ?
8642                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8643                         last_item = MLX5_FLOW_LAYER_MPLS;
8644                         break;
8645                 case RTE_FLOW_ITEM_TYPE_MARK:
8646                         flow_dv_translate_item_mark(dev, match_mask,
8647                                                     match_value, items);
8648                         last_item = MLX5_FLOW_ITEM_MARK;
8649                         break;
8650                 case RTE_FLOW_ITEM_TYPE_META:
8651                         flow_dv_translate_item_meta(dev, match_mask,
8652                                                     match_value, attr, items);
8653                         last_item = MLX5_FLOW_ITEM_METADATA;
8654                         break;
8655                 case RTE_FLOW_ITEM_TYPE_ICMP:
8656                         flow_dv_translate_item_icmp(match_mask, match_value,
8657                                                     items, tunnel);
8658                         last_item = MLX5_FLOW_LAYER_ICMP;
8659                         break;
8660                 case RTE_FLOW_ITEM_TYPE_ICMP6:
8661                         flow_dv_translate_item_icmp6(match_mask, match_value,
8662                                                       items, tunnel);
8663                         last_item = MLX5_FLOW_LAYER_ICMP6;
8664                         break;
8665                 case RTE_FLOW_ITEM_TYPE_TAG:
8666                         flow_dv_translate_item_tag(dev, match_mask,
8667                                                    match_value, items);
8668                         last_item = MLX5_FLOW_ITEM_TAG;
8669                         break;
8670                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8671                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
8672                                                         match_value, items);
8673                         last_item = MLX5_FLOW_ITEM_TAG;
8674                         break;
8675                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8676                         flow_dv_translate_item_tx_queue(dev, match_mask,
8677                                                         match_value,
8678                                                         items);
8679                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8680                         break;
8681                 case RTE_FLOW_ITEM_TYPE_GTP:
8682                         flow_dv_translate_item_gtp(match_mask, match_value,
8683                                                    items, tunnel);
8684                         matcher.priority = rss_desc->level >= 2 ?
8685                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8686                         last_item = MLX5_FLOW_LAYER_GTP;
8687                         break;
8688                 case RTE_FLOW_ITEM_TYPE_ECPRI:
8689                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
8690                                 /* Create it only the first time to be used. */
8691                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
8692                                 if (ret)
8693                                         return rte_flow_error_set
8694                                                 (error, -ret,
8695                                                 RTE_FLOW_ERROR_TYPE_ITEM,
8696                                                 NULL,
8697                                                 "cannot create eCPRI parser");
8698                         }
8699                         /* Adjust the length matcher and device flow value. */
8700                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
8701                         dev_flow->dv.value.size =
8702                                         MLX5_ST_SZ_BYTES(fte_match_param);
8703                         flow_dv_translate_item_ecpri(dev, match_mask,
8704                                                      match_value, items);
8705                         /* No other protocol should follow eCPRI layer. */
8706                         last_item = MLX5_FLOW_LAYER_ECPRI;
8707                         break;
8708                 default:
8709                         break;
8710                 }
8711                 item_flags |= last_item;
8712         }
8713         /*
8714          * When E-Switch mode is enabled, we have two cases where we need to
8715          * set the source port manually.
8716          * The first one, is in case of Nic steering rule, and the second is
8717          * E-Switch rule where no port_id item was found. In both cases
8718          * the source port is set according the current port in use.
8719          */
8720         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8721             (priv->representor || priv->master)) {
8722                 if (flow_dv_translate_item_port_id(dev, match_mask,
8723                                                    match_value, NULL))
8724                         return -rte_errno;
8725         }
8726 #ifdef RTE_LIBRTE_MLX5_DEBUG
8727         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8728                                               dev_flow->dv.value.buf));
8729 #endif
8730         /*
8731          * Layers may be already initialized from prefix flow if this dev_flow
8732          * is the suffix flow.
8733          */
8734         handle->layers |= item_flags;
8735         if (action_flags & MLX5_FLOW_ACTION_RSS)
8736                 flow_dv_hashfields_set(dev_flow, rss_desc);
8737         /* Register matcher. */
8738         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8739                                     matcher.mask.size);
8740         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8741                                                      matcher.priority);
8742         /* reserved field no needs to be set to 0 here. */
8743         tbl_key.domain = attr->transfer;
8744         tbl_key.direction = attr->egress;
8745         tbl_key.table_id = dev_flow->dv.group;
8746         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8747                 return -rte_errno;
8748         return 0;
8749 }
8750
8751 /**
8752  * Apply the flow to the NIC, lock free,
8753  * (mutex should be acquired by caller).
8754  *
8755  * @param[in] dev
8756  *   Pointer to the Ethernet device structure.
8757  * @param[in, out] flow
8758  *   Pointer to flow structure.
8759  * @param[out] error
8760  *   Pointer to error structure.
8761  *
8762  * @return
8763  *   0 on success, a negative errno value otherwise and rte_errno is set.
8764  */
8765 static int
8766 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8767                 struct rte_flow_error *error)
8768 {
8769         struct mlx5_flow_dv_workspace *dv;
8770         struct mlx5_flow_handle *dh;
8771         struct mlx5_flow_handle_dv *dv_h;
8772         struct mlx5_flow *dev_flow;
8773         struct mlx5_priv *priv = dev->data->dev_private;
8774         uint32_t handle_idx;
8775         int n;
8776         int err;
8777         int idx;
8778
8779         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8780                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8781                 dv = &dev_flow->dv;
8782                 dh = dev_flow->handle;
8783                 dv_h = &dh->dvh;
8784                 n = dv->actions_n;
8785                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8786                         if (dv->transfer) {
8787                                 dv->actions[n++] = priv->sh->esw_drop_action;
8788                         } else {
8789                                 struct mlx5_hrxq *drop_hrxq;
8790                                 drop_hrxq = mlx5_hrxq_drop_new(dev);
8791                                 if (!drop_hrxq) {
8792                                         rte_flow_error_set
8793                                                 (error, errno,
8794                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8795                                                  NULL,
8796                                                  "cannot get drop hash queue");
8797                                         goto error;
8798                                 }
8799                                 /*
8800                                  * Drop queues will be released by the specify
8801                                  * mlx5_hrxq_drop_release() function. Assign
8802                                  * the special index to hrxq to mark the queue
8803                                  * has been allocated.
8804                                  */
8805                                 dh->rix_hrxq = UINT32_MAX;
8806                                 dv->actions[n++] = drop_hrxq->action;
8807                         }
8808                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8809                         struct mlx5_hrxq *hrxq;
8810                         uint32_t hrxq_idx;
8811                         struct mlx5_flow_rss_desc *rss_desc =
8812                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8813                                 [!!priv->flow_nested_idx];
8814
8815                         MLX5_ASSERT(rss_desc->queue_num);
8816                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8817                                                  MLX5_RSS_HASH_KEY_LEN,
8818                                                  dev_flow->hash_fields,
8819                                                  rss_desc->queue,
8820                                                  rss_desc->queue_num);
8821                         if (!hrxq_idx) {
8822                                 hrxq_idx = mlx5_hrxq_new
8823                                                 (dev, rss_desc->key,
8824                                                 MLX5_RSS_HASH_KEY_LEN,
8825                                                 dev_flow->hash_fields,
8826                                                 rss_desc->queue,
8827                                                 rss_desc->queue_num,
8828                                                 !!(dh->layers &
8829                                                 MLX5_FLOW_LAYER_TUNNEL));
8830                         }
8831                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8832                                               hrxq_idx);
8833                         if (!hrxq) {
8834                                 rte_flow_error_set
8835                                         (error, rte_errno,
8836                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8837                                          "cannot get hash queue");
8838                                 goto error;
8839                         }
8840                         dh->rix_hrxq = hrxq_idx;
8841                         dv->actions[n++] = hrxq->action;
8842                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
8843                         if (flow_dv_default_miss_resource_register
8844                                         (dev, error)) {
8845                                 rte_flow_error_set
8846                                         (error, rte_errno,
8847                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8848                                          "cannot create default miss resource");
8849                                 goto error_default_miss;
8850                         }
8851                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
8852                         dv->actions[n++] = priv->sh->default_miss.action;
8853                 }
8854                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
8855                                                (void *)&dv->value, n,
8856                                                dv->actions, &dh->drv_flow);
8857                 if (err) {
8858                         rte_flow_error_set(error, errno,
8859                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8860                                            NULL,
8861                                            "hardware refuses to create flow");
8862                         goto error;
8863                 }
8864                 if (priv->vmwa_context &&
8865                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
8866                         /*
8867                          * The rule contains the VLAN pattern.
8868                          * For VF we are going to create VLAN
8869                          * interface to make hypervisor set correct
8870                          * e-Switch vport context.
8871                          */
8872                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8873                 }
8874         }
8875         return 0;
8876 error:
8877         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
8878                 flow_dv_default_miss_resource_release(dev);
8879 error_default_miss:
8880         err = rte_errno; /* Save rte_errno before cleanup. */
8881         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8882                        handle_idx, dh, next) {
8883                 /* hrxq is union, don't clear it if the flag is not set. */
8884                 if (dh->rix_hrxq) {
8885                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8886                                 mlx5_hrxq_drop_release(dev);
8887                                 dh->rix_hrxq = 0;
8888                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8889                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
8890                                 dh->rix_hrxq = 0;
8891                         }
8892                 }
8893                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8894                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8895         }
8896         rte_errno = err; /* Restore rte_errno. */
8897         return -rte_errno;
8898 }
8899
8900 /**
8901  * Release the flow matcher.
8902  *
8903  * @param dev
8904  *   Pointer to Ethernet device.
8905  * @param handle
8906  *   Pointer to mlx5_flow_handle.
8907  *
8908  * @return
8909  *   1 while a reference on it exists, 0 when freed.
8910  */
8911 static int
8912 flow_dv_matcher_release(struct rte_eth_dev *dev,
8913                         struct mlx5_flow_handle *handle)
8914 {
8915         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8916
8917         MLX5_ASSERT(matcher->matcher_object);
8918         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8919                 dev->data->port_id, (void *)matcher,
8920                 rte_atomic32_read(&matcher->refcnt));
8921         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8922                 claim_zero(mlx5_flow_os_destroy_flow_matcher
8923                            (matcher->matcher_object));
8924                 LIST_REMOVE(matcher, next);
8925                 /* table ref-- in release interface. */
8926                 flow_dv_tbl_resource_release(dev, matcher->tbl);
8927                 mlx5_free(matcher);
8928                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8929                         dev->data->port_id, (void *)matcher);
8930                 return 0;
8931         }
8932         return 1;
8933 }
8934
8935 /**
8936  * Release an encap/decap resource.
8937  *
8938  * @param dev
8939  *   Pointer to Ethernet device.
8940  * @param handle
8941  *   Pointer to mlx5_flow_handle.
8942  *
8943  * @return
8944  *   1 while a reference on it exists, 0 when freed.
8945  */
8946 static int
8947 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8948                                      struct mlx5_flow_handle *handle)
8949 {
8950         struct mlx5_priv *priv = dev->data->dev_private;
8951         uint32_t idx = handle->dvh.rix_encap_decap;
8952         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8953
8954         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8955                          idx);
8956         if (!cache_resource)
8957                 return 0;
8958         MLX5_ASSERT(cache_resource->action);
8959         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8960                 (void *)cache_resource,
8961                 rte_atomic32_read(&cache_resource->refcnt));
8962         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8963                 claim_zero(mlx5_flow_os_destroy_flow_action
8964                                                 (cache_resource->action));
8965                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8966                              &priv->sh->encaps_decaps, idx,
8967                              cache_resource, next);
8968                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8969                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8970                         (void *)cache_resource);
8971                 return 0;
8972         }
8973         return 1;
8974 }
8975
8976 /**
8977  * Release an jump to table action resource.
8978  *
8979  * @param dev
8980  *   Pointer to Ethernet device.
8981  * @param handle
8982  *   Pointer to mlx5_flow_handle.
8983  *
8984  * @return
8985  *   1 while a reference on it exists, 0 when freed.
8986  */
8987 static int
8988 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8989                                   struct mlx5_flow_handle *handle)
8990 {
8991         struct mlx5_priv *priv = dev->data->dev_private;
8992         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8993         struct mlx5_flow_tbl_data_entry *tbl_data;
8994
8995         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8996                              handle->rix_jump);
8997         if (!tbl_data)
8998                 return 0;
8999         cache_resource = &tbl_data->jump;
9000         MLX5_ASSERT(cache_resource->action);
9001         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
9002                 (void *)cache_resource,
9003                 rte_atomic32_read(&cache_resource->refcnt));
9004         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9005                 claim_zero(mlx5_flow_os_destroy_flow_action
9006                                                 (cache_resource->action));
9007                 /* jump action memory free is inside the table release. */
9008                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
9009                 DRV_LOG(DEBUG, "jump table resource %p: removed",
9010                         (void *)cache_resource);
9011                 return 0;
9012         }
9013         return 1;
9014 }
9015
9016 /**
9017  * Release a default miss resource.
9018  *
9019  * @param dev
9020  *   Pointer to Ethernet device.
9021  * @return
9022  *   1 while a reference on it exists, 0 when freed.
9023  */
9024 static int
9025 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
9026 {
9027         struct mlx5_priv *priv = dev->data->dev_private;
9028         struct mlx5_dev_ctx_shared *sh = priv->sh;
9029         struct mlx5_flow_default_miss_resource *cache_resource =
9030                         &sh->default_miss;
9031
9032         MLX5_ASSERT(cache_resource->action);
9033         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
9034                         (void *)cache_resource->action,
9035                         rte_atomic32_read(&cache_resource->refcnt));
9036         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9037                 claim_zero(mlx5_glue->destroy_flow_action
9038                                 (cache_resource->action));
9039                 DRV_LOG(DEBUG, "default miss resource %p: removed",
9040                                 (void *)cache_resource->action);
9041                 return 0;
9042         }
9043         return 1;
9044 }
9045
9046 /**
9047  * Release a modify-header resource.
9048  *
9049  * @param handle
9050  *   Pointer to mlx5_flow_handle.
9051  *
9052  * @return
9053  *   1 while a reference on it exists, 0 when freed.
9054  */
9055 static int
9056 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
9057 {
9058         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
9059                                                         handle->dvh.modify_hdr;
9060
9061         MLX5_ASSERT(cache_resource->action);
9062         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
9063                 (void *)cache_resource,
9064                 rte_atomic32_read(&cache_resource->refcnt));
9065         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9066                 claim_zero(mlx5_flow_os_destroy_flow_action
9067                                                 (cache_resource->action));
9068                 LIST_REMOVE(cache_resource, next);
9069                 mlx5_free(cache_resource);
9070                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
9071                         (void *)cache_resource);
9072                 return 0;
9073         }
9074         return 1;
9075 }
9076
9077 /**
9078  * Release port ID action resource.
9079  *
9080  * @param dev
9081  *   Pointer to Ethernet device.
9082  * @param handle
9083  *   Pointer to mlx5_flow_handle.
9084  *
9085  * @return
9086  *   1 while a reference on it exists, 0 when freed.
9087  */
9088 static int
9089 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
9090                                         struct mlx5_flow_handle *handle)
9091 {
9092         struct mlx5_priv *priv = dev->data->dev_private;
9093         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
9094         uint32_t idx = handle->rix_port_id_action;
9095
9096         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
9097                                         idx);
9098         if (!cache_resource)
9099                 return 0;
9100         MLX5_ASSERT(cache_resource->action);
9101         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
9102                 (void *)cache_resource,
9103                 rte_atomic32_read(&cache_resource->refcnt));
9104         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9105                 claim_zero(mlx5_flow_os_destroy_flow_action
9106                                                 (cache_resource->action));
9107                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
9108                              &priv->sh->port_id_action_list, idx,
9109                              cache_resource, next);
9110                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
9111                 DRV_LOG(DEBUG, "port id action resource %p: removed",
9112                         (void *)cache_resource);
9113                 return 0;
9114         }
9115         return 1;
9116 }
9117
9118 /**
9119  * Release push vlan action resource.
9120  *
9121  * @param dev
9122  *   Pointer to Ethernet device.
9123  * @param handle
9124  *   Pointer to mlx5_flow_handle.
9125  *
9126  * @return
9127  *   1 while a reference on it exists, 0 when freed.
9128  */
9129 static int
9130 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
9131                                           struct mlx5_flow_handle *handle)
9132 {
9133         struct mlx5_priv *priv = dev->data->dev_private;
9134         uint32_t idx = handle->dvh.rix_push_vlan;
9135         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
9136
9137         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9138                                         idx);
9139         if (!cache_resource)
9140                 return 0;
9141         MLX5_ASSERT(cache_resource->action);
9142         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
9143                 (void *)cache_resource,
9144                 rte_atomic32_read(&cache_resource->refcnt));
9145         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9146                 claim_zero(mlx5_flow_os_destroy_flow_action
9147                                                 (cache_resource->action));
9148                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9149                              &priv->sh->push_vlan_action_list, idx,
9150                              cache_resource, next);
9151                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
9152                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
9153                         (void *)cache_resource);
9154                 return 0;
9155         }
9156         return 1;
9157 }
9158
9159 /**
9160  * Release the fate resource.
9161  *
9162  * @param dev
9163  *   Pointer to Ethernet device.
9164  * @param handle
9165  *   Pointer to mlx5_flow_handle.
9166  */
9167 static void
9168 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
9169                                struct mlx5_flow_handle *handle)
9170 {
9171         if (!handle->rix_fate)
9172                 return;
9173         switch (handle->fate_action) {
9174         case MLX5_FLOW_FATE_DROP:
9175                 mlx5_hrxq_drop_release(dev);
9176                 break;
9177         case MLX5_FLOW_FATE_QUEUE:
9178                 mlx5_hrxq_release(dev, handle->rix_hrxq);
9179                 break;
9180         case MLX5_FLOW_FATE_JUMP:
9181                 flow_dv_jump_tbl_resource_release(dev, handle);
9182                 break;
9183         case MLX5_FLOW_FATE_PORT_ID:
9184                 flow_dv_port_id_action_resource_release(dev, handle);
9185                 break;
9186         case MLX5_FLOW_FATE_DEFAULT_MISS:
9187                 flow_dv_default_miss_resource_release(dev);
9188                 break;
9189         default:
9190                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9191                 break;
9192         }
9193         handle->rix_fate = 0;
9194 }
9195
9196 /**
9197  * Remove the flow from the NIC but keeps it in memory.
9198  * Lock free, (mutex should be acquired by caller).
9199  *
9200  * @param[in] dev
9201  *   Pointer to Ethernet device.
9202  * @param[in, out] flow
9203  *   Pointer to flow structure.
9204  */
9205 static void
9206 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9207 {
9208         struct mlx5_flow_handle *dh;
9209         uint32_t handle_idx;
9210         struct mlx5_priv *priv = dev->data->dev_private;
9211
9212         if (!flow)
9213                 return;
9214         handle_idx = flow->dev_handles;
9215         while (handle_idx) {
9216                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9217                                     handle_idx);
9218                 if (!dh)
9219                         return;
9220                 if (dh->drv_flow) {
9221                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
9222                         dh->drv_flow = NULL;
9223                 }
9224                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9225                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9226                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9227                         flow_dv_fate_resource_release(dev, dh);
9228                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9229                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9230                 handle_idx = dh->next.next;
9231         }
9232 }
9233
9234 /**
9235  * Remove the flow from the NIC and the memory.
9236  * Lock free, (mutex should be acquired by caller).
9237  *
9238  * @param[in] dev
9239  *   Pointer to the Ethernet device structure.
9240  * @param[in, out] flow
9241  *   Pointer to flow structure.
9242  */
9243 static void
9244 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9245 {
9246         struct mlx5_flow_handle *dev_handle;
9247         struct mlx5_priv *priv = dev->data->dev_private;
9248
9249         if (!flow)
9250                 return;
9251         __flow_dv_remove(dev, flow);
9252         if (flow->counter) {
9253                 flow_dv_counter_release(dev, flow->counter);
9254                 flow->counter = 0;
9255         }
9256         if (flow->meter) {
9257                 struct mlx5_flow_meter *fm;
9258
9259                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9260                                     flow->meter);
9261                 if (fm)
9262                         mlx5_flow_meter_detach(fm);
9263                 flow->meter = 0;
9264         }
9265         while (flow->dev_handles) {
9266                 uint32_t tmp_idx = flow->dev_handles;
9267
9268                 dev_handle = mlx5_ipool_get(priv->sh->ipool
9269                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9270                 if (!dev_handle)
9271                         return;
9272                 flow->dev_handles = dev_handle->next.next;
9273                 if (dev_handle->dvh.matcher)
9274                         flow_dv_matcher_release(dev, dev_handle);
9275                 if (dev_handle->dvh.rix_encap_decap)
9276                         flow_dv_encap_decap_resource_release(dev, dev_handle);
9277                 if (dev_handle->dvh.modify_hdr)
9278                         flow_dv_modify_hdr_resource_release(dev_handle);
9279                 if (dev_handle->dvh.rix_push_vlan)
9280                         flow_dv_push_vlan_action_resource_release(dev,
9281                                                                   dev_handle);
9282                 if (dev_handle->dvh.rix_tag)
9283                         flow_dv_tag_release(dev,
9284                                             dev_handle->dvh.rix_tag);
9285                 flow_dv_fate_resource_release(dev, dev_handle);
9286                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9287                            tmp_idx);
9288         }
9289 }
9290
9291 /**
9292  * Query a dv flow  rule for its statistics via devx.
9293  *
9294  * @param[in] dev
9295  *   Pointer to Ethernet device.
9296  * @param[in] flow
9297  *   Pointer to the sub flow.
9298  * @param[out] data
9299  *   data retrieved by the query.
9300  * @param[out] error
9301  *   Perform verbose error reporting if not NULL.
9302  *
9303  * @return
9304  *   0 on success, a negative errno value otherwise and rte_errno is set.
9305  */
9306 static int
9307 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9308                     void *data, struct rte_flow_error *error)
9309 {
9310         struct mlx5_priv *priv = dev->data->dev_private;
9311         struct rte_flow_query_count *qc = data;
9312
9313         if (!priv->config.devx)
9314                 return rte_flow_error_set(error, ENOTSUP,
9315                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9316                                           NULL,
9317                                           "counters are not supported");
9318         if (flow->counter) {
9319                 uint64_t pkts, bytes;
9320                 struct mlx5_flow_counter *cnt;
9321
9322                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9323                                                  NULL);
9324                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9325                                                &bytes);
9326
9327                 if (err)
9328                         return rte_flow_error_set(error, -err,
9329                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9330                                         NULL, "cannot read counters");
9331                 qc->hits_set = 1;
9332                 qc->bytes_set = 1;
9333                 qc->hits = pkts - cnt->hits;
9334                 qc->bytes = bytes - cnt->bytes;
9335                 if (qc->reset) {
9336                         cnt->hits = pkts;
9337                         cnt->bytes = bytes;
9338                 }
9339                 return 0;
9340         }
9341         return rte_flow_error_set(error, EINVAL,
9342                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9343                                   NULL,
9344                                   "counters are not available");
9345 }
9346
9347 /**
9348  * Query a flow.
9349  *
9350  * @see rte_flow_query()
9351  * @see rte_flow_ops
9352  */
9353 static int
9354 flow_dv_query(struct rte_eth_dev *dev,
9355               struct rte_flow *flow __rte_unused,
9356               const struct rte_flow_action *actions __rte_unused,
9357               void *data __rte_unused,
9358               struct rte_flow_error *error __rte_unused)
9359 {
9360         int ret = -EINVAL;
9361
9362         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9363                 switch (actions->type) {
9364                 case RTE_FLOW_ACTION_TYPE_VOID:
9365                         break;
9366                 case RTE_FLOW_ACTION_TYPE_COUNT:
9367                         ret = flow_dv_query_count(dev, flow, data, error);
9368                         break;
9369                 default:
9370                         return rte_flow_error_set(error, ENOTSUP,
9371                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9372                                                   actions,
9373                                                   "action not supported");
9374                 }
9375         }
9376         return ret;
9377 }
9378
9379 /**
9380  * Destroy the meter table set.
9381  * Lock free, (mutex should be acquired by caller).
9382  *
9383  * @param[in] dev
9384  *   Pointer to Ethernet device.
9385  * @param[in] tbl
9386  *   Pointer to the meter table set.
9387  *
9388  * @return
9389  *   Always 0.
9390  */
9391 static int
9392 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9393                         struct mlx5_meter_domains_infos *tbl)
9394 {
9395         struct mlx5_priv *priv = dev->data->dev_private;
9396         struct mlx5_meter_domains_infos *mtd =
9397                                 (struct mlx5_meter_domains_infos *)tbl;
9398
9399         if (!mtd || !priv->config.dv_flow_en)
9400                 return 0;
9401         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9402                 claim_zero(mlx5_flow_os_destroy_flow
9403                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9404         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9405                 claim_zero(mlx5_flow_os_destroy_flow
9406                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9407         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9408                 claim_zero(mlx5_flow_os_destroy_flow
9409                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9410         if (mtd->egress.color_matcher)
9411                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9412                            (mtd->egress.color_matcher));
9413         if (mtd->egress.any_matcher)
9414                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9415                            (mtd->egress.any_matcher));
9416         if (mtd->egress.tbl)
9417                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9418         if (mtd->egress.sfx_tbl)
9419                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9420         if (mtd->ingress.color_matcher)
9421                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9422                            (mtd->ingress.color_matcher));
9423         if (mtd->ingress.any_matcher)
9424                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9425                            (mtd->ingress.any_matcher));
9426         if (mtd->ingress.tbl)
9427                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9428         if (mtd->ingress.sfx_tbl)
9429                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9430         if (mtd->transfer.color_matcher)
9431                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9432                            (mtd->transfer.color_matcher));
9433         if (mtd->transfer.any_matcher)
9434                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9435                            (mtd->transfer.any_matcher));
9436         if (mtd->transfer.tbl)
9437                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9438         if (mtd->transfer.sfx_tbl)
9439                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9440         if (mtd->drop_actn)
9441                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
9442         mlx5_free(mtd);
9443         return 0;
9444 }
9445
9446 /* Number of meter flow actions, count and jump or count and drop. */
9447 #define METER_ACTIONS 2
9448
9449 /**
9450  * Create specify domain meter table and suffix table.
9451  *
9452  * @param[in] dev
9453  *   Pointer to Ethernet device.
9454  * @param[in,out] mtb
9455  *   Pointer to DV meter table set.
9456  * @param[in] egress
9457  *   Table attribute.
9458  * @param[in] transfer
9459  *   Table attribute.
9460  * @param[in] color_reg_c_idx
9461  *   Reg C index for color match.
9462  *
9463  * @return
9464  *   0 on success, -1 otherwise and rte_errno is set.
9465  */
9466 static int
9467 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9468                            struct mlx5_meter_domains_infos *mtb,
9469                            uint8_t egress, uint8_t transfer,
9470                            uint32_t color_reg_c_idx)
9471 {
9472         struct mlx5_priv *priv = dev->data->dev_private;
9473         struct mlx5_dev_ctx_shared *sh = priv->sh;
9474         struct mlx5_flow_dv_match_params mask = {
9475                 .size = sizeof(mask.buf),
9476         };
9477         struct mlx5_flow_dv_match_params value = {
9478                 .size = sizeof(value.buf),
9479         };
9480         struct mlx5dv_flow_matcher_attr dv_attr = {
9481                 .type = IBV_FLOW_ATTR_NORMAL,
9482                 .priority = 0,
9483                 .match_criteria_enable = 0,
9484                 .match_mask = (void *)&mask,
9485         };
9486         void *actions[METER_ACTIONS];
9487         struct mlx5_meter_domain_info *dtb;
9488         struct rte_flow_error error;
9489         int i = 0;
9490         int ret;
9491
9492         if (transfer)
9493                 dtb = &mtb->transfer;
9494         else if (egress)
9495                 dtb = &mtb->egress;
9496         else
9497                 dtb = &mtb->ingress;
9498         /* Create the meter table with METER level. */
9499         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9500                                             egress, transfer, &error);
9501         if (!dtb->tbl) {
9502                 DRV_LOG(ERR, "Failed to create meter policer table.");
9503                 return -1;
9504         }
9505         /* Create the meter suffix table with SUFFIX level. */
9506         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9507                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9508                                             egress, transfer, &error);
9509         if (!dtb->sfx_tbl) {
9510                 DRV_LOG(ERR, "Failed to create meter suffix table.");
9511                 return -1;
9512         }
9513         /* Create matchers, Any and Color. */
9514         dv_attr.priority = 3;
9515         dv_attr.match_criteria_enable = 0;
9516         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9517                                                &dtb->any_matcher);
9518         if (ret) {
9519                 DRV_LOG(ERR, "Failed to create meter"
9520                              " policer default matcher.");
9521                 goto error_exit;
9522         }
9523         dv_attr.priority = 0;
9524         dv_attr.match_criteria_enable =
9525                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9526         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9527                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9528         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9529                                                &dtb->color_matcher);
9530         if (ret) {
9531                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9532                 goto error_exit;
9533         }
9534         if (mtb->count_actns[RTE_MTR_DROPPED])
9535                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9536         actions[i++] = mtb->drop_actn;
9537         /* Default rule: lowest priority, match any, actions: drop. */
9538         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
9539                                        actions,
9540                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
9541         if (ret) {
9542                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9543                 goto error_exit;
9544         }
9545         return 0;
9546 error_exit:
9547         return -1;
9548 }
9549
9550 /**
9551  * Create the needed meter and suffix tables.
9552  * Lock free, (mutex should be acquired by caller).
9553  *
9554  * @param[in] dev
9555  *   Pointer to Ethernet device.
9556  * @param[in] fm
9557  *   Pointer to the flow meter.
9558  *
9559  * @return
9560  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
9561  */
9562 static struct mlx5_meter_domains_infos *
9563 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9564                        const struct mlx5_flow_meter *fm)
9565 {
9566         struct mlx5_priv *priv = dev->data->dev_private;
9567         struct mlx5_meter_domains_infos *mtb;
9568         int ret;
9569         int i;
9570
9571         if (!priv->mtr_en) {
9572                 rte_errno = ENOTSUP;
9573                 return NULL;
9574         }
9575         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
9576         if (!mtb) {
9577                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9578                 return NULL;
9579         }
9580         /* Create meter count actions */
9581         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9582                 struct mlx5_flow_counter *cnt;
9583                 if (!fm->policer_stats.cnt[i])
9584                         continue;
9585                 cnt = flow_dv_counter_get_by_idx(dev,
9586                       fm->policer_stats.cnt[i], NULL);
9587                 mtb->count_actns[i] = cnt->action;
9588         }
9589         /* Create drop action. */
9590         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
9591         if (ret) {
9592                 DRV_LOG(ERR, "Failed to create drop action.");
9593                 goto error_exit;
9594         }
9595         /* Egress meter table. */
9596         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9597         if (ret) {
9598                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9599                 goto error_exit;
9600         }
9601         /* Ingress meter table. */
9602         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9603         if (ret) {
9604                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9605                 goto error_exit;
9606         }
9607         /* FDB meter table. */
9608         if (priv->config.dv_esw_en) {
9609                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9610                                                  priv->mtr_color_reg);
9611                 if (ret) {
9612                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9613                         goto error_exit;
9614                 }
9615         }
9616         return mtb;
9617 error_exit:
9618         flow_dv_destroy_mtr_tbl(dev, mtb);
9619         return NULL;
9620 }
9621
9622 /**
9623  * Destroy domain policer rule.
9624  *
9625  * @param[in] dt
9626  *   Pointer to domain table.
9627  */
9628 static void
9629 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9630 {
9631         int i;
9632
9633         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9634                 if (dt->policer_rules[i]) {
9635                         claim_zero(mlx5_flow_os_destroy_flow
9636                                    (dt->policer_rules[i]));
9637                         dt->policer_rules[i] = NULL;
9638                 }
9639         }
9640         if (dt->jump_actn) {
9641                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
9642                 dt->jump_actn = NULL;
9643         }
9644 }
9645
9646 /**
9647  * Destroy policer rules.
9648  *
9649  * @param[in] dev
9650  *   Pointer to Ethernet device.
9651  * @param[in] fm
9652  *   Pointer to flow meter structure.
9653  * @param[in] attr
9654  *   Pointer to flow attributes.
9655  *
9656  * @return
9657  *   Always 0.
9658  */
9659 static int
9660 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9661                               const struct mlx5_flow_meter *fm,
9662                               const struct rte_flow_attr *attr)
9663 {
9664         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9665
9666         if (!mtb)
9667                 return 0;
9668         if (attr->egress)
9669                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9670         if (attr->ingress)
9671                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9672         if (attr->transfer)
9673                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9674         return 0;
9675 }
9676
9677 /**
9678  * Create specify domain meter policer rule.
9679  *
9680  * @param[in] fm
9681  *   Pointer to flow meter structure.
9682  * @param[in] mtb
9683  *   Pointer to DV meter table set.
9684  * @param[in] mtr_reg_c
9685  *   Color match REG_C.
9686  *
9687  * @return
9688  *   0 on success, -1 otherwise.
9689  */
9690 static int
9691 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9692                                     struct mlx5_meter_domain_info *dtb,
9693                                     uint8_t mtr_reg_c)
9694 {
9695         struct mlx5_flow_dv_match_params matcher = {
9696                 .size = sizeof(matcher.buf),
9697         };
9698         struct mlx5_flow_dv_match_params value = {
9699                 .size = sizeof(value.buf),
9700         };
9701         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9702         void *actions[METER_ACTIONS];
9703         int i;
9704         int ret = 0;
9705
9706         /* Create jump action. */
9707         if (!dtb->jump_actn)
9708                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
9709                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
9710         if (ret) {
9711                 DRV_LOG(ERR, "Failed to create policer jump action.");
9712                 goto error;
9713         }
9714         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9715                 int j = 0;
9716
9717                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9718                                        rte_col_2_mlx5_col(i), UINT8_MAX);
9719                 if (mtb->count_actns[i])
9720                         actions[j++] = mtb->count_actns[i];
9721                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9722                         actions[j++] = mtb->drop_actn;
9723                 else
9724                         actions[j++] = dtb->jump_actn;
9725                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
9726                                                (void *)&value, j, actions,
9727                                                &dtb->policer_rules[i]);
9728                 if (ret) {
9729                         DRV_LOG(ERR, "Failed to create policer rule.");
9730                         goto error;
9731                 }
9732         }
9733         return 0;
9734 error:
9735         rte_errno = errno;
9736         return -1;
9737 }
9738
9739 /**
9740  * Create policer rules.
9741  *
9742  * @param[in] dev
9743  *   Pointer to Ethernet device.
9744  * @param[in] fm
9745  *   Pointer to flow meter structure.
9746  * @param[in] attr
9747  *   Pointer to flow attributes.
9748  *
9749  * @return
9750  *   0 on success, -1 otherwise.
9751  */
9752 static int
9753 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9754                              struct mlx5_flow_meter *fm,
9755                              const struct rte_flow_attr *attr)
9756 {
9757         struct mlx5_priv *priv = dev->data->dev_private;
9758         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9759         int ret;
9760
9761         if (attr->egress) {
9762                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9763                                                 priv->mtr_color_reg);
9764                 if (ret) {
9765                         DRV_LOG(ERR, "Failed to create egress policer.");
9766                         goto error;
9767                 }
9768         }
9769         if (attr->ingress) {
9770                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9771                                                 priv->mtr_color_reg);
9772                 if (ret) {
9773                         DRV_LOG(ERR, "Failed to create ingress policer.");
9774                         goto error;
9775                 }
9776         }
9777         if (attr->transfer) {
9778                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9779                                                 priv->mtr_color_reg);
9780                 if (ret) {
9781                         DRV_LOG(ERR, "Failed to create transfer policer.");
9782                         goto error;
9783                 }
9784         }
9785         return 0;
9786 error:
9787         flow_dv_destroy_policer_rules(dev, fm, attr);
9788         return -1;
9789 }
9790
9791 /**
9792  * Query a devx counter.
9793  *
9794  * @param[in] dev
9795  *   Pointer to the Ethernet device structure.
9796  * @param[in] cnt
9797  *   Index to the flow counter.
9798  * @param[in] clear
9799  *   Set to clear the counter statistics.
9800  * @param[out] pkts
9801  *   The statistics value of packets.
9802  * @param[out] bytes
9803  *   The statistics value of bytes.
9804  *
9805  * @return
9806  *   0 on success, otherwise return -1.
9807  */
9808 static int
9809 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9810                       uint64_t *pkts, uint64_t *bytes)
9811 {
9812         struct mlx5_priv *priv = dev->data->dev_private;
9813         struct mlx5_flow_counter *cnt;
9814         uint64_t inn_pkts, inn_bytes;
9815         int ret;
9816
9817         if (!priv->config.devx)
9818                 return -1;
9819
9820         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9821         if (ret)
9822                 return -1;
9823         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9824         *pkts = inn_pkts - cnt->hits;
9825         *bytes = inn_bytes - cnt->bytes;
9826         if (clear) {
9827                 cnt->hits = inn_pkts;
9828                 cnt->bytes = inn_bytes;
9829         }
9830         return 0;
9831 }
9832
9833 /**
9834  * Get aged-out flows.
9835  *
9836  * @param[in] dev
9837  *   Pointer to the Ethernet device structure.
9838  * @param[in] context
9839  *   The address of an array of pointers to the aged-out flows contexts.
9840  * @param[in] nb_contexts
9841  *   The length of context array pointers.
9842  * @param[out] error
9843  *   Perform verbose error reporting if not NULL. Initialized in case of
9844  *   error only.
9845  *
9846  * @return
9847  *   how many contexts get in success, otherwise negative errno value.
9848  *   if nb_contexts is 0, return the amount of all aged contexts.
9849  *   if nb_contexts is not 0 , return the amount of aged flows reported
9850  *   in the context array.
9851  * @note: only stub for now
9852  */
9853 static int
9854 flow_get_aged_flows(struct rte_eth_dev *dev,
9855                     void **context,
9856                     uint32_t nb_contexts,
9857                     struct rte_flow_error *error)
9858 {
9859         struct mlx5_priv *priv = dev->data->dev_private;
9860         struct mlx5_age_info *age_info;
9861         struct mlx5_age_param *age_param;
9862         struct mlx5_flow_counter *counter;
9863         int nb_flows = 0;
9864
9865         if (nb_contexts && !context)
9866                 return rte_flow_error_set(error, EINVAL,
9867                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9868                                           NULL,
9869                                           "Should assign at least one flow or"
9870                                           " context to get if nb_contexts != 0");
9871         age_info = GET_PORT_AGE_INFO(priv);
9872         rte_spinlock_lock(&age_info->aged_sl);
9873         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9874                 nb_flows++;
9875                 if (nb_contexts) {
9876                         age_param = MLX5_CNT_TO_AGE(counter);
9877                         context[nb_flows - 1] = age_param->context;
9878                         if (!(--nb_contexts))
9879                                 break;
9880                 }
9881         }
9882         rte_spinlock_unlock(&age_info->aged_sl);
9883         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9884         return nb_flows;
9885 }
9886
9887 /*
9888  * Mutex-protected thunk to lock-free  __flow_dv_translate().
9889  */
9890 static int
9891 flow_dv_translate(struct rte_eth_dev *dev,
9892                   struct mlx5_flow *dev_flow,
9893                   const struct rte_flow_attr *attr,
9894                   const struct rte_flow_item items[],
9895                   const struct rte_flow_action actions[],
9896                   struct rte_flow_error *error)
9897 {
9898         int ret;
9899
9900         flow_dv_shared_lock(dev);
9901         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9902         flow_dv_shared_unlock(dev);
9903         return ret;
9904 }
9905
9906 /*
9907  * Mutex-protected thunk to lock-free  __flow_dv_apply().
9908  */
9909 static int
9910 flow_dv_apply(struct rte_eth_dev *dev,
9911               struct rte_flow *flow,
9912               struct rte_flow_error *error)
9913 {
9914         int ret;
9915
9916         flow_dv_shared_lock(dev);
9917         ret = __flow_dv_apply(dev, flow, error);
9918         flow_dv_shared_unlock(dev);
9919         return ret;
9920 }
9921
9922 /*
9923  * Mutex-protected thunk to lock-free __flow_dv_remove().
9924  */
9925 static void
9926 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9927 {
9928         flow_dv_shared_lock(dev);
9929         __flow_dv_remove(dev, flow);
9930         flow_dv_shared_unlock(dev);
9931 }
9932
9933 /*
9934  * Mutex-protected thunk to lock-free __flow_dv_destroy().
9935  */
9936 static void
9937 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9938 {
9939         flow_dv_shared_lock(dev);
9940         __flow_dv_destroy(dev, flow);
9941         flow_dv_shared_unlock(dev);
9942 }
9943
9944 /*
9945  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9946  */
9947 static uint32_t
9948 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9949 {
9950         uint32_t cnt;
9951
9952         flow_dv_shared_lock(dev);
9953         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9954         flow_dv_shared_unlock(dev);
9955         return cnt;
9956 }
9957
9958 /*
9959  * Mutex-protected thunk to lock-free flow_dv_counter_release().
9960  */
9961 static void
9962 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9963 {
9964         flow_dv_shared_lock(dev);
9965         flow_dv_counter_release(dev, cnt);
9966         flow_dv_shared_unlock(dev);
9967 }
9968
9969 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9970         .validate = flow_dv_validate,
9971         .prepare = flow_dv_prepare,
9972         .translate = flow_dv_translate,
9973         .apply = flow_dv_apply,
9974         .remove = flow_dv_remove,
9975         .destroy = flow_dv_destroy,
9976         .query = flow_dv_query,
9977         .create_mtr_tbls = flow_dv_create_mtr_tbl,
9978         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9979         .create_policer_rules = flow_dv_create_policer_rules,
9980         .destroy_policer_rules = flow_dv_destroy_policer_rules,
9981         .counter_alloc = flow_dv_counter_allocate,
9982         .counter_free = flow_dv_counter_free,
9983         .counter_query = flow_dv_counter_query,
9984         .get_aged_flows = flow_get_aged_flows,
9985 };
9986
9987 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */