drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29 #include <rte_vxlan.h>
  30
  31 #include "mlx5.h"
  32 #include "mlx5_defs.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35 #include "mlx5_prm.h"
  36 #include "mlx5_rxtx.h"
  37
  38 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  39
  40 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  41 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  42 #endif
  43
  44 #ifndef HAVE_MLX5DV_DR_ESWITCH
  45 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  46 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  47 #endif
  48 #endif
  49
  50 #ifndef HAVE_MLX5DV_DR
  51 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  52 #endif
  53
  54 /* VLAN header definitions */
  55 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  56 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  57 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  58 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  59 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  60
  61 union flow_dv_attr {
  62         struct {
  63                 uint32_t valid:1;
  64                 uint32_t ipv4:1;
  65                 uint32_t ipv6:1;
  66                 uint32_t tcp:1;
  67                 uint32_t udp:1;
  68                 uint32_t reserved:27;
  69         };
  70         uint32_t attr;
  71 };
  72
  73 /**
  74  * Initialize flow attributes structure according to flow items' types.
  75  *
  76  * @param[in] item
  77  *   Pointer to item specification.
  78  * @param[out] attr
  79  *   Pointer to flow attributes structure.
  80  */
  81 static void
  82 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  83 {
  84         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  85                 switch (item->type) {
  86                 case RTE_FLOW_ITEM_TYPE_IPV4:
  87                         attr->ipv4 = 1;
  88                         break;
  89                 case RTE_FLOW_ITEM_TYPE_IPV6:
  90                         attr->ipv6 = 1;
  91                         break;
  92                 case RTE_FLOW_ITEM_TYPE_UDP:
  93                         attr->udp = 1;
  94                         break;
  95                 case RTE_FLOW_ITEM_TYPE_TCP:
  96                         attr->tcp = 1;
  97                         break;
  98                 default:
  99                         break;
 100                 }
 101         }
 102         attr->valid = 1;
 103 }
 104
 105 /**
 106  * Convert rte_mtr_color to mlx5 color.
 107  *
 108  * @param[in] rcol
 109  *   rte_mtr_color.
 110  *
 111  * @return
 112  *   mlx5 color.
 113  */
 114 static int
 115 rte_col_2_mlx5_col(enum rte_color rcol)
 116 {
 117         switch (rcol) {
 118         case RTE_COLOR_GREEN:
 119                 return MLX5_FLOW_COLOR_GREEN;
 120         case RTE_COLOR_YELLOW:
 121                 return MLX5_FLOW_COLOR_YELLOW;
 122         case RTE_COLOR_RED:
 123                 return MLX5_FLOW_COLOR_RED;
 124         default:
 125                 break;
 126         }
 127         return MLX5_FLOW_COLOR_UNDEFINED;
 128 }
 129
 130 struct field_modify_info {
 131         uint32_t size; /* Size of field in protocol header, in bytes. */
 132         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 133         enum mlx5_modification_field id;
 134 };
 135
 136 struct field_modify_info modify_eth[] = {
 137         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 138         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 139         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 140         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 141         {0, 0, 0},
 142 };
 143
 144 struct field_modify_info modify_vlan_out_first_vid[] = {
 145         /* Size in bits !!! */
 146         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 147         {0, 0, 0},
 148 };
 149
 150 struct field_modify_info modify_ipv4[] = {
 151         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 152         {4, 12, MLX5_MODI_OUT_SIPV4},
 153         {4, 16, MLX5_MODI_OUT_DIPV4},
 154         {0, 0, 0},
 155 };
 156
 157 struct field_modify_info modify_ipv6[] = {
 158         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 159         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 160         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 161         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 162         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 163         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 164         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 165         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 166         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 167         {0, 0, 0},
 168 };
 169
 170 struct field_modify_info modify_udp[] = {
 171         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 172         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 173         {0, 0, 0},
 174 };
 175
 176 struct field_modify_info modify_tcp[] = {
 177         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 178         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 179         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 180         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 181         {0, 0, 0},
 182 };
 183
 184 static void
 185 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 186                           uint8_t next_protocol, uint64_t *item_flags,
 187                           int *tunnel)
 188 {
 189         assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 190                item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 191         if (next_protocol == IPPROTO_IPIP) {
 192                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 193                 *tunnel = 1;
 194         }
 195         if (next_protocol == IPPROTO_IPV6) {
 196                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 197                 *tunnel = 1;
 198         }
 199 }
 200
 201 /**
 202  * Acquire the synchronizing object to protect multithreaded access
 203  * to shared dv context. Lock occurs only if context is actually
 204  * shared, i.e. we have multiport IB device and representors are
 205  * created.
 206  *
 207  * @param[in] dev
 208  *   Pointer to the rte_eth_dev structure.
 209  */
 210 static void
 211 flow_dv_shared_lock(struct rte_eth_dev *dev)
 212 {
 213         struct mlx5_priv *priv = dev->data->dev_private;
 214         struct mlx5_ibv_shared *sh = priv->sh;
 215
 216         if (sh->dv_refcnt > 1) {
 217                 int ret;
 218
 219                 ret = pthread_mutex_lock(&sh->dv_mutex);
 220                 assert(!ret);
 221                 (void)ret;
 222         }
 223 }
 224
 225 static void
 226 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 227 {
 228         struct mlx5_priv *priv = dev->data->dev_private;
 229         struct mlx5_ibv_shared *sh = priv->sh;
 230
 231         if (sh->dv_refcnt > 1) {
 232                 int ret;
 233
 234                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 235                 assert(!ret);
 236                 (void)ret;
 237         }
 238 }
 239
 240 /* Update VLAN's VID/PCP based on input rte_flow_action.
 241  *
 242  * @param[in] action
 243  *   Pointer to struct rte_flow_action.
 244  * @param[out] vlan
 245  *   Pointer to struct rte_vlan_hdr.
 246  */
 247 static void
 248 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 249                          struct rte_vlan_hdr *vlan)
 250 {
 251         uint16_t vlan_tci;
 252         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 253                 vlan_tci =
 254                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 255                                                action->conf)->vlan_pcp;
 256                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 257                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 258                 vlan->vlan_tci |= vlan_tci;
 259         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 260                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 261                 vlan->vlan_tci |= rte_be_to_cpu_16
 262                     (((const struct rte_flow_action_of_set_vlan_vid *)
 263                                              action->conf)->vlan_vid);
 264         }
 265 }
 266
 267 /**
 268  * Fetch 1, 2, 3 or 4 byte field from the byte array
 269  * and return as unsigned integer in host-endian format.
 270  *
 271  * @param[in] data
 272  *   Pointer to data array.
 273  * @param[in] size
 274  *   Size of field to extract.
 275  *
 276  * @return
 277  *   converted field in host endian format.
 278  */
 279 static inline uint32_t
 280 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 281 {
 282         uint32_t ret;
 283
 284         switch (size) {
 285         case 1:
 286                 ret = *data;
 287                 break;
 288         case 2:
 289                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 290                 break;
 291         case 3:
 292                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 293                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 294                 break;
 295         case 4:
 296                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 297                 break;
 298         default:
 299                 assert(false);
 300                 ret = 0;
 301                 break;
 302         }
 303         return ret;
 304 }
 305
 306 /**
 307  * Convert modify-header action to DV specification.
 308  *
 309  * Data length of each action is determined by provided field description
 310  * and the item mask. Data bit offset and width of each action is determined
 311  * by provided item mask.
 312  *
 313  * @param[in] item
 314  *   Pointer to item specification.
 315  * @param[in] field
 316  *   Pointer to field modification information.
 317  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 318  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 319  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 320  * @param[in] dcopy
 321  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 322  *   Negative offset value sets the same offset as source offset.
 323  *   size field is ignored, value is taken from source field.
 324  * @param[in,out] resource
 325  *   Pointer to the modify-header resource.
 326  * @param[in] type
 327  *   Type of modification.
 328  * @param[out] error
 329  *   Pointer to the error structure.
 330  *
 331  * @return
 332  *   0 on success, a negative errno value otherwise and rte_errno is set.
 333  */
 334 static int
 335 flow_dv_convert_modify_action(struct rte_flow_item *item,
 336                               struct field_modify_info *field,
 337                               struct field_modify_info *dcopy,
 338                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 339                               uint32_t type, struct rte_flow_error *error)
 340 {
 341         uint32_t i = resource->actions_num;
 342         struct mlx5_modification_cmd *actions = resource->actions;
 343
 344         /*
 345          * The item and mask are provided in big-endian format.
 346          * The fields should be presented as in big-endian format either.
 347          * Mask must be always present, it defines the actual field width.
 348          */
 349         assert(item->mask);
 350         assert(field->size);
 351         do {
 352                 unsigned int size_b;
 353                 unsigned int off_b;
 354                 uint32_t mask;
 355                 uint32_t data;
 356
 357                 if (i >= MLX5_MODIFY_NUM)
 358                         return rte_flow_error_set(error, EINVAL,
 359                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 360                                  "too many items to modify");
 361                 /* Fetch variable byte size mask from the array. */
 362                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 363                                            field->offset, field->size);
 364                 if (!mask) {
 365                         ++field;
 366                         continue;
 367                 }
 368                 /* Deduce actual data width in bits from mask value. */
 369                 off_b = rte_bsf32(mask);
 370                 size_b = sizeof(uint32_t) * CHAR_BIT -
 371                          off_b - __builtin_clz(mask);
 372                 assert(size_b);
 373                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 374                 actions[i].action_type = type;
 375                 actions[i].field = field->id;
 376                 actions[i].offset = off_b;
 377                 actions[i].length = size_b;
 378                 /* Convert entire record to expected big-endian format. */
 379                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 380                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 381                         assert(dcopy);
 382                         actions[i].dst_field = dcopy->id;
 383                         actions[i].dst_offset =
 384                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 385                         /* Convert entire record to big-endian format. */
 386                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 387                 } else {
 388                         assert(item->spec);
 389                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 390                                                    field->offset, field->size);
 391                         /* Shift out the trailing masked bits from data. */
 392                         data = (data & mask) >> off_b;
 393                         actions[i].data1 = rte_cpu_to_be_32(data);
 394                 }
 395                 ++i;
 396                 ++field;
 397         } while (field->size);
 398         resource->actions_num = i;
 399         if (!resource->actions_num)
 400                 return rte_flow_error_set(error, EINVAL,
 401                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 402                                           "invalid modification flow item");
 403         return 0;
 404 }
 405
 406 /**
 407  * Convert modify-header set IPv4 address action to DV specification.
 408  *
 409  * @param[in,out] resource
 410  *   Pointer to the modify-header resource.
 411  * @param[in] action
 412  *   Pointer to action specification.
 413  * @param[out] error
 414  *   Pointer to the error structure.
 415  *
 416  * @return
 417  *   0 on success, a negative errno value otherwise and rte_errno is set.
 418  */
 419 static int
 420 flow_dv_convert_action_modify_ipv4
 421                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 422                          const struct rte_flow_action *action,
 423                          struct rte_flow_error *error)
 424 {
 425         const struct rte_flow_action_set_ipv4 *conf =
 426                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 427         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 428         struct rte_flow_item_ipv4 ipv4;
 429         struct rte_flow_item_ipv4 ipv4_mask;
 430
 431         memset(&ipv4, 0, sizeof(ipv4));
 432         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 433         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 434                 ipv4.hdr.src_addr = conf->ipv4_addr;
 435                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 436         } else {
 437                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 438                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 439         }
 440         item.spec = &ipv4;
 441         item.mask = &ipv4_mask;
 442         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 443                                              MLX5_MODIFICATION_TYPE_SET, error);
 444 }
 445
 446 /**
 447  * Convert modify-header set IPv6 address action to DV specification.
 448  *
 449  * @param[in,out] resource
 450  *   Pointer to the modify-header resource.
 451  * @param[in] action
 452  *   Pointer to action specification.
 453  * @param[out] error
 454  *   Pointer to the error structure.
 455  *
 456  * @return
 457  *   0 on success, a negative errno value otherwise and rte_errno is set.
 458  */
 459 static int
 460 flow_dv_convert_action_modify_ipv6
 461                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 462                          const struct rte_flow_action *action,
 463                          struct rte_flow_error *error)
 464 {
 465         const struct rte_flow_action_set_ipv6 *conf =
 466                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 467         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 468         struct rte_flow_item_ipv6 ipv6;
 469         struct rte_flow_item_ipv6 ipv6_mask;
 470
 471         memset(&ipv6, 0, sizeof(ipv6));
 472         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 473         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 474                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 475                        sizeof(ipv6.hdr.src_addr));
 476                 memcpy(&ipv6_mask.hdr.src_addr,
 477                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 478                        sizeof(ipv6.hdr.src_addr));
 479         } else {
 480                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 481                        sizeof(ipv6.hdr.dst_addr));
 482                 memcpy(&ipv6_mask.hdr.dst_addr,
 483                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 484                        sizeof(ipv6.hdr.dst_addr));
 485         }
 486         item.spec = &ipv6;
 487         item.mask = &ipv6_mask;
 488         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 489                                              MLX5_MODIFICATION_TYPE_SET, error);
 490 }
 491
 492 /**
 493  * Convert modify-header set MAC address action to DV specification.
 494  *
 495  * @param[in,out] resource
 496  *   Pointer to the modify-header resource.
 497  * @param[in] action
 498  *   Pointer to action specification.
 499  * @param[out] error
 500  *   Pointer to the error structure.
 501  *
 502  * @return
 503  *   0 on success, a negative errno value otherwise and rte_errno is set.
 504  */
 505 static int
 506 flow_dv_convert_action_modify_mac
 507                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 508                          const struct rte_flow_action *action,
 509                          struct rte_flow_error *error)
 510 {
 511         const struct rte_flow_action_set_mac *conf =
 512                 (const struct rte_flow_action_set_mac *)(action->conf);
 513         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 514         struct rte_flow_item_eth eth;
 515         struct rte_flow_item_eth eth_mask;
 516
 517         memset(&eth, 0, sizeof(eth));
 518         memset(&eth_mask, 0, sizeof(eth_mask));
 519         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 520                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 521                        sizeof(eth.src.addr_bytes));
 522                 memcpy(&eth_mask.src.addr_bytes,
 523                        &rte_flow_item_eth_mask.src.addr_bytes,
 524                        sizeof(eth_mask.src.addr_bytes));
 525         } else {
 526                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 527                        sizeof(eth.dst.addr_bytes));
 528                 memcpy(&eth_mask.dst.addr_bytes,
 529                        &rte_flow_item_eth_mask.dst.addr_bytes,
 530                        sizeof(eth_mask.dst.addr_bytes));
 531         }
 532         item.spec = &eth;
 533         item.mask = &eth_mask;
 534         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 535                                              MLX5_MODIFICATION_TYPE_SET, error);
 536 }
 537
 538 /**
 539  * Convert modify-header set VLAN VID action to DV specification.
 540  *
 541  * @param[in,out] resource
 542  *   Pointer to the modify-header resource.
 543  * @param[in] action
 544  *   Pointer to action specification.
 545  * @param[out] error
 546  *   Pointer to the error structure.
 547  *
 548  * @return
 549  *   0 on success, a negative errno value otherwise and rte_errno is set.
 550  */
 551 static int
 552 flow_dv_convert_action_modify_vlan_vid
 553                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 554                          const struct rte_flow_action *action,
 555                          struct rte_flow_error *error)
 556 {
 557         const struct rte_flow_action_of_set_vlan_vid *conf =
 558                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 559         int i = resource->actions_num;
 560         struct mlx5_modification_cmd *actions = &resource->actions[i];
 561         struct field_modify_info *field = modify_vlan_out_first_vid;
 562
 563         if (i >= MLX5_MODIFY_NUM)
 564                 return rte_flow_error_set(error, EINVAL,
 565                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 566                          "too many items to modify");
 567         actions[i].action_type = MLX5_MODIFICATION_TYPE_SET;
 568         actions[i].field = field->id;
 569         actions[i].length = field->size;
 570         actions[i].offset = field->offset;
 571         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 572         actions[i].data1 = conf->vlan_vid;
 573         actions[i].data1 = actions[i].data1 << 16;
 574         resource->actions_num = ++i;
 575         return 0;
 576 }
 577
 578 /**
 579  * Convert modify-header set TP action to DV specification.
 580  *
 581  * @param[in,out] resource
 582  *   Pointer to the modify-header resource.
 583  * @param[in] action
 584  *   Pointer to action specification.
 585  * @param[in] items
 586  *   Pointer to rte_flow_item objects list.
 587  * @param[in] attr
 588  *   Pointer to flow attributes structure.
 589  * @param[out] error
 590  *   Pointer to the error structure.
 591  *
 592  * @return
 593  *   0 on success, a negative errno value otherwise and rte_errno is set.
 594  */
 595 static int
 596 flow_dv_convert_action_modify_tp
 597                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 598                          const struct rte_flow_action *action,
 599                          const struct rte_flow_item *items,
 600                          union flow_dv_attr *attr,
 601                          struct rte_flow_error *error)
 602 {
 603         const struct rte_flow_action_set_tp *conf =
 604                 (const struct rte_flow_action_set_tp *)(action->conf);
 605         struct rte_flow_item item;
 606         struct rte_flow_item_udp udp;
 607         struct rte_flow_item_udp udp_mask;
 608         struct rte_flow_item_tcp tcp;
 609         struct rte_flow_item_tcp tcp_mask;
 610         struct field_modify_info *field;
 611
 612         if (!attr->valid)
 613                 flow_dv_attr_init(items, attr);
 614         if (attr->udp) {
 615                 memset(&udp, 0, sizeof(udp));
 616                 memset(&udp_mask, 0, sizeof(udp_mask));
 617                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 618                         udp.hdr.src_port = conf->port;
 619                         udp_mask.hdr.src_port =
 620                                         rte_flow_item_udp_mask.hdr.src_port;
 621                 } else {
 622                         udp.hdr.dst_port = conf->port;
 623                         udp_mask.hdr.dst_port =
 624                                         rte_flow_item_udp_mask.hdr.dst_port;
 625                 }
 626                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 627                 item.spec = &udp;
 628                 item.mask = &udp_mask;
 629                 field = modify_udp;
 630         }
 631         if (attr->tcp) {
 632                 memset(&tcp, 0, sizeof(tcp));
 633                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 634                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 635                         tcp.hdr.src_port = conf->port;
 636                         tcp_mask.hdr.src_port =
 637                                         rte_flow_item_tcp_mask.hdr.src_port;
 638                 } else {
 639                         tcp.hdr.dst_port = conf->port;
 640                         tcp_mask.hdr.dst_port =
 641                                         rte_flow_item_tcp_mask.hdr.dst_port;
 642                 }
 643                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 644                 item.spec = &tcp;
 645                 item.mask = &tcp_mask;
 646                 field = modify_tcp;
 647         }
 648         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 649                                              MLX5_MODIFICATION_TYPE_SET, error);
 650 }
 651
 652 /**
 653  * Convert modify-header set TTL action to DV specification.
 654  *
 655  * @param[in,out] resource
 656  *   Pointer to the modify-header resource.
 657  * @param[in] action
 658  *   Pointer to action specification.
 659  * @param[in] items
 660  *   Pointer to rte_flow_item objects list.
 661  * @param[in] attr
 662  *   Pointer to flow attributes structure.
 663  * @param[out] error
 664  *   Pointer to the error structure.
 665  *
 666  * @return
 667  *   0 on success, a negative errno value otherwise and rte_errno is set.
 668  */
 669 static int
 670 flow_dv_convert_action_modify_ttl
 671                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 672                          const struct rte_flow_action *action,
 673                          const struct rte_flow_item *items,
 674                          union flow_dv_attr *attr,
 675                          struct rte_flow_error *error)
 676 {
 677         const struct rte_flow_action_set_ttl *conf =
 678                 (const struct rte_flow_action_set_ttl *)(action->conf);
 679         struct rte_flow_item item;
 680         struct rte_flow_item_ipv4 ipv4;
 681         struct rte_flow_item_ipv4 ipv4_mask;
 682         struct rte_flow_item_ipv6 ipv6;
 683         struct rte_flow_item_ipv6 ipv6_mask;
 684         struct field_modify_info *field;
 685
 686         if (!attr->valid)
 687                 flow_dv_attr_init(items, attr);
 688         if (attr->ipv4) {
 689                 memset(&ipv4, 0, sizeof(ipv4));
 690                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 691                 ipv4.hdr.time_to_live = conf->ttl_value;
 692                 ipv4_mask.hdr.time_to_live = 0xFF;
 693                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 694                 item.spec = &ipv4;
 695                 item.mask = &ipv4_mask;
 696                 field = modify_ipv4;
 697         }
 698         if (attr->ipv6) {
 699                 memset(&ipv6, 0, sizeof(ipv6));
 700                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 701                 ipv6.hdr.hop_limits = conf->ttl_value;
 702                 ipv6_mask.hdr.hop_limits = 0xFF;
 703                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 704                 item.spec = &ipv6;
 705                 item.mask = &ipv6_mask;
 706                 field = modify_ipv6;
 707         }
 708         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 709                                              MLX5_MODIFICATION_TYPE_SET, error);
 710 }
 711
 712 /**
 713  * Convert modify-header decrement TTL action to DV specification.
 714  *
 715  * @param[in,out] resource
 716  *   Pointer to the modify-header resource.
 717  * @param[in] action
 718  *   Pointer to action specification.
 719  * @param[in] items
 720  *   Pointer to rte_flow_item objects list.
 721  * @param[in] attr
 722  *   Pointer to flow attributes structure.
 723  * @param[out] error
 724  *   Pointer to the error structure.
 725  *
 726  * @return
 727  *   0 on success, a negative errno value otherwise and rte_errno is set.
 728  */
 729 static int
 730 flow_dv_convert_action_modify_dec_ttl
 731                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 732                          const struct rte_flow_item *items,
 733                          union flow_dv_attr *attr,
 734                          struct rte_flow_error *error)
 735 {
 736         struct rte_flow_item item;
 737         struct rte_flow_item_ipv4 ipv4;
 738         struct rte_flow_item_ipv4 ipv4_mask;
 739         struct rte_flow_item_ipv6 ipv6;
 740         struct rte_flow_item_ipv6 ipv6_mask;
 741         struct field_modify_info *field;
 742
 743         if (!attr->valid)
 744                 flow_dv_attr_init(items, attr);
 745         if (attr->ipv4) {
 746                 memset(&ipv4, 0, sizeof(ipv4));
 747                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 748                 ipv4.hdr.time_to_live = 0xFF;
 749                 ipv4_mask.hdr.time_to_live = 0xFF;
 750                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 751                 item.spec = &ipv4;
 752                 item.mask = &ipv4_mask;
 753                 field = modify_ipv4;
 754         }
 755         if (attr->ipv6) {
 756                 memset(&ipv6, 0, sizeof(ipv6));
 757                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 758                 ipv6.hdr.hop_limits = 0xFF;
 759                 ipv6_mask.hdr.hop_limits = 0xFF;
 760                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 761                 item.spec = &ipv6;
 762                 item.mask = &ipv6_mask;
 763                 field = modify_ipv6;
 764         }
 765         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 766                                              MLX5_MODIFICATION_TYPE_ADD, error);
 767 }
 768
 769 /**
 770  * Convert modify-header increment/decrement TCP Sequence number
 771  * to DV specification.
 772  *
 773  * @param[in,out] resource
 774  *   Pointer to the modify-header resource.
 775  * @param[in] action
 776  *   Pointer to action specification.
 777  * @param[out] error
 778  *   Pointer to the error structure.
 779  *
 780  * @return
 781  *   0 on success, a negative errno value otherwise and rte_errno is set.
 782  */
 783 static int
 784 flow_dv_convert_action_modify_tcp_seq
 785                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 786                          const struct rte_flow_action *action,
 787                          struct rte_flow_error *error)
 788 {
 789         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 790         uint64_t value = rte_be_to_cpu_32(*conf);
 791         struct rte_flow_item item;
 792         struct rte_flow_item_tcp tcp;
 793         struct rte_flow_item_tcp tcp_mask;
 794
 795         memset(&tcp, 0, sizeof(tcp));
 796         memset(&tcp_mask, 0, sizeof(tcp_mask));
 797         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 798                 /*
 799                  * The HW has no decrement operation, only increment operation.
 800                  * To simulate decrement X from Y using increment operation
 801                  * we need to add UINT32_MAX X times to Y.
 802                  * Each adding of UINT32_MAX decrements Y by 1.
 803                  */
 804                 value *= UINT32_MAX;
 805         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 806         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 807         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 808         item.spec = &tcp;
 809         item.mask = &tcp_mask;
 810         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 811                                              MLX5_MODIFICATION_TYPE_ADD, error);
 812 }
 813
 814 /**
 815  * Convert modify-header increment/decrement TCP Acknowledgment number
 816  * to DV specification.
 817  *
 818  * @param[in,out] resource
 819  *   Pointer to the modify-header resource.
 820  * @param[in] action
 821  *   Pointer to action specification.
 822  * @param[out] error
 823  *   Pointer to the error structure.
 824  *
 825  * @return
 826  *   0 on success, a negative errno value otherwise and rte_errno is set.
 827  */
 828 static int
 829 flow_dv_convert_action_modify_tcp_ack
 830                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 831                          const struct rte_flow_action *action,
 832                          struct rte_flow_error *error)
 833 {
 834         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 835         uint64_t value = rte_be_to_cpu_32(*conf);
 836         struct rte_flow_item item;
 837         struct rte_flow_item_tcp tcp;
 838         struct rte_flow_item_tcp tcp_mask;
 839
 840         memset(&tcp, 0, sizeof(tcp));
 841         memset(&tcp_mask, 0, sizeof(tcp_mask));
 842         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 843                 /*
 844                  * The HW has no decrement operation, only increment operation.
 845                  * To simulate decrement X from Y using increment operation
 846                  * we need to add UINT32_MAX X times to Y.
 847                  * Each adding of UINT32_MAX decrements Y by 1.
 848                  */
 849                 value *= UINT32_MAX;
 850         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 851         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 852         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 853         item.spec = &tcp;
 854         item.mask = &tcp_mask;
 855         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 856                                              MLX5_MODIFICATION_TYPE_ADD, error);
 857 }
 858
 859 static enum mlx5_modification_field reg_to_field[] = {
 860         [REG_NONE] = MLX5_MODI_OUT_NONE,
 861         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 862         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 863         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 864         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 865         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 866         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 867         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 868         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 869         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 870         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 871 };
 872
 873 /**
 874  * Convert register set to DV specification.
 875  *
 876  * @param[in,out] resource
 877  *   Pointer to the modify-header resource.
 878  * @param[in] action
 879  *   Pointer to action specification.
 880  * @param[out] error
 881  *   Pointer to the error structure.
 882  *
 883  * @return
 884  *   0 on success, a negative errno value otherwise and rte_errno is set.
 885  */
 886 static int
 887 flow_dv_convert_action_set_reg
 888                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 889                          const struct rte_flow_action *action,
 890                          struct rte_flow_error *error)
 891 {
 892         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 893         struct mlx5_modification_cmd *actions = resource->actions;
 894         uint32_t i = resource->actions_num;
 895
 896         if (i >= MLX5_MODIFY_NUM)
 897                 return rte_flow_error_set(error, EINVAL,
 898                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 899                                           "too many items to modify");
 900         assert(conf->id != REG_NONE);
 901         assert(conf->id < RTE_DIM(reg_to_field));
 902         actions[i].action_type = MLX5_MODIFICATION_TYPE_SET;
 903         actions[i].field = reg_to_field[conf->id];
 904         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 905         actions[i].data1 = rte_cpu_to_be_32(conf->data);
 906         ++i;
 907         resource->actions_num = i;
 908         if (!resource->actions_num)
 909                 return rte_flow_error_set(error, EINVAL,
 910                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 911                                           "invalid modification flow item");
 912         return 0;
 913 }
 914
 915 /**
 916  * Convert SET_TAG action to DV specification.
 917  *
 918  * @param[in] dev
 919  *   Pointer to the rte_eth_dev structure.
 920  * @param[in,out] resource
 921  *   Pointer to the modify-header resource.
 922  * @param[in] conf
 923  *   Pointer to action specification.
 924  * @param[out] error
 925  *   Pointer to the error structure.
 926  *
 927  * @return
 928  *   0 on success, a negative errno value otherwise and rte_errno is set.
 929  */
 930 static int
 931 flow_dv_convert_action_set_tag
 932                         (struct rte_eth_dev *dev,
 933                          struct mlx5_flow_dv_modify_hdr_resource *resource,
 934                          const struct rte_flow_action_set_tag *conf,
 935                          struct rte_flow_error *error)
 936 {
 937         rte_be32_t data = rte_cpu_to_be_32(conf->data);
 938         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
 939         struct rte_flow_item item = {
 940                 .spec = &data,
 941                 .mask = &mask,
 942         };
 943         struct field_modify_info reg_c_x[] = {
 944                 [1] = {0, 0, 0},
 945         };
 946         enum mlx5_modification_field reg_type;
 947         int ret;
 948
 949         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
 950         if (ret < 0)
 951                 return ret;
 952         assert(ret != REG_NONE);
 953         assert((unsigned int)ret < RTE_DIM(reg_to_field));
 954         reg_type = reg_to_field[ret];
 955         assert(reg_type > 0);
 956         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
 957         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
 958                                              MLX5_MODIFICATION_TYPE_SET, error);
 959 }
 960
 961 /**
 962  * Convert internal COPY_REG action to DV specification.
 963  *
 964  * @param[in] dev
 965  *   Pointer to the rte_eth_dev structure.
 966  * @param[in,out] res
 967  *   Pointer to the modify-header resource.
 968  * @param[in] action
 969  *   Pointer to action specification.
 970  * @param[out] error
 971  *   Pointer to the error structure.
 972  *
 973  * @return
 974  *   0 on success, a negative errno value otherwise and rte_errno is set.
 975  */
 976 static int
 977 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
 978                                  struct mlx5_flow_dv_modify_hdr_resource *res,
 979                                  const struct rte_flow_action *action,
 980                                  struct rte_flow_error *error)
 981 {
 982         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
 983         rte_be32_t mask = RTE_BE32(UINT32_MAX);
 984         struct rte_flow_item item = {
 985                 .spec = NULL,
 986                 .mask = &mask,
 987         };
 988         struct field_modify_info reg_src[] = {
 989                 {4, 0, reg_to_field[conf->src]},
 990                 {0, 0, 0},
 991         };
 992         struct field_modify_info reg_dst = {
 993                 .offset = 0,
 994                 .id = reg_to_field[conf->dst],
 995         };
 996         /* Adjust reg_c[0] usage according to reported mask. */
 997         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
 998                 struct mlx5_priv *priv = dev->data->dev_private;
 999                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1000
1001                 assert(reg_c0);
1002                 assert(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1003                 if (conf->dst == REG_C_0) {
1004                         /* Copy to reg_c[0], within mask only. */
1005                         reg_dst.offset = rte_bsf32(reg_c0);
1006                         /*
1007                          * Mask is ignoring the enianness, because
1008                          * there is no conversion in datapath.
1009                          */
1010 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1011                         /* Copy from destination lower bits to reg_c[0]. */
1012                         mask = reg_c0 >> reg_dst.offset;
1013 #else
1014                         /* Copy from destination upper bits to reg_c[0]. */
1015                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1016                                           rte_fls_u32(reg_c0));
1017 #endif
1018                 } else {
1019                         mask = rte_cpu_to_be_32(reg_c0);
1020 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1021                         /* Copy from reg_c[0] to destination lower bits. */
1022                         reg_dst.offset = 0;
1023 #else
1024                         /* Copy from reg_c[0] to destination upper bits. */
1025                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1026                                          (rte_fls_u32(reg_c0) -
1027                                           rte_bsf32(reg_c0));
1028 #endif
1029                 }
1030         }
1031         return flow_dv_convert_modify_action(&item,
1032                                              reg_src, &reg_dst, res,
1033                                              MLX5_MODIFICATION_TYPE_COPY,
1034                                              error);
1035 }
1036
1037 /**
1038  * Convert MARK action to DV specification. This routine is used
1039  * in extensive metadata only and requires metadata register to be
1040  * handled. In legacy mode hardware tag resource is engaged.
1041  *
1042  * @param[in] dev
1043  *   Pointer to the rte_eth_dev structure.
1044  * @param[in] conf
1045  *   Pointer to MARK action specification.
1046  * @param[in,out] resource
1047  *   Pointer to the modify-header resource.
1048  * @param[out] error
1049  *   Pointer to the error structure.
1050  *
1051  * @return
1052  *   0 on success, a negative errno value otherwise and rte_errno is set.
1053  */
1054 static int
1055 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1056                             const struct rte_flow_action_mark *conf,
1057                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1058                             struct rte_flow_error *error)
1059 {
1060         struct mlx5_priv *priv = dev->data->dev_private;
1061         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1062                                            priv->sh->dv_mark_mask);
1063         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1064         struct rte_flow_item item = {
1065                 .spec = &data,
1066                 .mask = &mask,
1067         };
1068         struct field_modify_info reg_c_x[] = {
1069                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1070                 {0, 0, 0},
1071         };
1072         enum modify_reg reg;
1073
1074         if (!mask)
1075                 return rte_flow_error_set(error, EINVAL,
1076                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1077                                           NULL, "zero mark action mask");
1078         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1079         if (reg < 0)
1080                 return reg;
1081         assert(reg > 0);
1082         reg_c_x[0].id = reg_to_field[reg];
1083         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1084                                              MLX5_MODIFICATION_TYPE_SET, error);
1085 }
1086
1087 /**
1088  * Get metadata register index for specified steering domain.
1089  *
1090  * @param[in] dev
1091  *   Pointer to the rte_eth_dev structure.
1092  * @param[in] attr
1093  *   Attributes of flow to determine steering domain.
1094  * @param[out] error
1095  *   Pointer to the error structure.
1096  *
1097  * @return
1098  *   positive index on success, a negative errno value otherwise
1099  *   and rte_errno is set.
1100  */
1101 static enum modify_reg
1102 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1103                          const struct rte_flow_attr *attr,
1104                          struct rte_flow_error *error)
1105 {
1106         enum modify_reg reg =
1107                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1108                                           MLX5_METADATA_FDB :
1109                                             attr->egress ?
1110                                             MLX5_METADATA_TX :
1111                                             MLX5_METADATA_RX, 0, error);
1112         if (reg < 0)
1113                 return rte_flow_error_set(error,
1114                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1115                                           NULL, "unavailable "
1116                                           "metadata register");
1117         return reg;
1118 }
1119
1120 /**
1121  * Convert SET_META action to DV specification.
1122  *
1123  * @param[in] dev
1124  *   Pointer to the rte_eth_dev structure.
1125  * @param[in,out] resource
1126  *   Pointer to the modify-header resource.
1127  * @param[in] attr
1128  *   Attributes of flow that includes this item.
1129  * @param[in] conf
1130  *   Pointer to action specification.
1131  * @param[out] error
1132  *   Pointer to the error structure.
1133  *
1134  * @return
1135  *   0 on success, a negative errno value otherwise and rte_errno is set.
1136  */
1137 static int
1138 flow_dv_convert_action_set_meta
1139                         (struct rte_eth_dev *dev,
1140                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1141                          const struct rte_flow_attr *attr,
1142                          const struct rte_flow_action_set_meta *conf,
1143                          struct rte_flow_error *error)
1144 {
1145         uint32_t data = conf->data;
1146         uint32_t mask = conf->mask;
1147         struct rte_flow_item item = {
1148                 .spec = &data,
1149                 .mask = &mask,
1150         };
1151         struct field_modify_info reg_c_x[] = {
1152                 [1] = {0, 0, 0},
1153         };
1154         enum modify_reg reg = flow_dv_get_metadata_reg(dev, attr, error);
1155
1156         if (reg < 0)
1157                 return reg;
1158         /*
1159          * In datapath code there is no endianness
1160          * coversions for perfromance reasons, all
1161          * pattern conversions are done in rte_flow.
1162          */
1163         if (reg == REG_C_0) {
1164                 struct mlx5_priv *priv = dev->data->dev_private;
1165                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1166                 uint32_t shl_c0;
1167
1168                 assert(msk_c0);
1169 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1170                 shl_c0 = rte_bsf32(msk_c0);
1171 #else
1172                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1173 #endif
1174                 mask <<= shl_c0;
1175                 data <<= shl_c0;
1176                 assert(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1177         }
1178         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1179         /* The routine expects parameters in memory as big-endian ones. */
1180         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1181                                              MLX5_MODIFICATION_TYPE_SET, error);
1182 }
1183
1184 /**
1185  * Validate MARK item.
1186  *
1187  * @param[in] dev
1188  *   Pointer to the rte_eth_dev structure.
1189  * @param[in] item
1190  *   Item specification.
1191  * @param[in] attr
1192  *   Attributes of flow that includes this item.
1193  * @param[out] error
1194  *   Pointer to error structure.
1195  *
1196  * @return
1197  *   0 on success, a negative errno value otherwise and rte_errno is set.
1198  */
1199 static int
1200 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1201                            const struct rte_flow_item *item,
1202                            const struct rte_flow_attr *attr __rte_unused,
1203                            struct rte_flow_error *error)
1204 {
1205         struct mlx5_priv *priv = dev->data->dev_private;
1206         struct mlx5_dev_config *config = &priv->config;
1207         const struct rte_flow_item_mark *spec = item->spec;
1208         const struct rte_flow_item_mark *mask = item->mask;
1209         const struct rte_flow_item_mark nic_mask = {
1210                 .id = priv->sh->dv_mark_mask,
1211         };
1212         int ret;
1213
1214         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1215                 return rte_flow_error_set(error, ENOTSUP,
1216                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1217                                           "extended metadata feature"
1218                                           " isn't enabled");
1219         if (!mlx5_flow_ext_mreg_supported(dev))
1220                 return rte_flow_error_set(error, ENOTSUP,
1221                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1222                                           "extended metadata register"
1223                                           " isn't supported");
1224         if (!nic_mask.id)
1225                 return rte_flow_error_set(error, ENOTSUP,
1226                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1227                                           "extended metadata register"
1228                                           " isn't available");
1229         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1230         if (ret < 0)
1231                 return ret;
1232         if (!spec)
1233                 return rte_flow_error_set(error, EINVAL,
1234                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1235                                           item->spec,
1236                                           "data cannot be empty");
1237         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1238                 return rte_flow_error_set(error, EINVAL,
1239                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1240                                           &spec->id,
1241                                           "mark id exceeds the limit");
1242         if (!mask)
1243                 mask = &nic_mask;
1244         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1245                                         (const uint8_t *)&nic_mask,
1246                                         sizeof(struct rte_flow_item_mark),
1247                                         error);
1248         if (ret < 0)
1249                 return ret;
1250         return 0;
1251 }
1252
1253 /**
1254  * Validate META item.
1255  *
1256  * @param[in] dev
1257  *   Pointer to the rte_eth_dev structure.
1258  * @param[in] item
1259  *   Item specification.
1260  * @param[in] attr
1261  *   Attributes of flow that includes this item.
1262  * @param[out] error
1263  *   Pointer to error structure.
1264  *
1265  * @return
1266  *   0 on success, a negative errno value otherwise and rte_errno is set.
1267  */
1268 static int
1269 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1270                            const struct rte_flow_item *item,
1271                            const struct rte_flow_attr *attr,
1272                            struct rte_flow_error *error)
1273 {
1274         struct mlx5_priv *priv = dev->data->dev_private;
1275         struct mlx5_dev_config *config = &priv->config;
1276         const struct rte_flow_item_meta *spec = item->spec;
1277         const struct rte_flow_item_meta *mask = item->mask;
1278         struct rte_flow_item_meta nic_mask = {
1279                 .data = UINT32_MAX
1280         };
1281         enum modify_reg reg;
1282         int ret;
1283
1284         if (!spec)
1285                 return rte_flow_error_set(error, EINVAL,
1286                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1287                                           item->spec,
1288                                           "data cannot be empty");
1289         if (!spec->data)
1290                 return rte_flow_error_set(error, EINVAL,
1291                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1292                                           "data cannot be zero");
1293         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1294                 if (!mlx5_flow_ext_mreg_supported(dev))
1295                         return rte_flow_error_set(error, ENOTSUP,
1296                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1297                                           "extended metadata register"
1298                                           " isn't supported");
1299                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1300                 if (reg < 0)
1301                         return reg;
1302                 if (reg == REG_B)
1303                         return rte_flow_error_set(error, ENOTSUP,
1304                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1305                                           "match on reg_b "
1306                                           "isn't supported");
1307                 if (reg != REG_A)
1308                         nic_mask.data = priv->sh->dv_meta_mask;
1309         }
1310         if (!mask)
1311                 mask = &rte_flow_item_meta_mask;
1312         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1313                                         (const uint8_t *)&nic_mask,
1314                                         sizeof(struct rte_flow_item_meta),
1315                                         error);
1316         return ret;
1317 }
1318
1319 /**
1320  * Validate TAG item.
1321  *
1322  * @param[in] dev
1323  *   Pointer to the rte_eth_dev structure.
1324  * @param[in] item
1325  *   Item specification.
1326  * @param[in] attr
1327  *   Attributes of flow that includes this item.
1328  * @param[out] error
1329  *   Pointer to error structure.
1330  *
1331  * @return
1332  *   0 on success, a negative errno value otherwise and rte_errno is set.
1333  */
1334 static int
1335 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1336                           const struct rte_flow_item *item,
1337                           const struct rte_flow_attr *attr __rte_unused,
1338                           struct rte_flow_error *error)
1339 {
1340         const struct rte_flow_item_tag *spec = item->spec;
1341         const struct rte_flow_item_tag *mask = item->mask;
1342         const struct rte_flow_item_tag nic_mask = {
1343                 .data = RTE_BE32(UINT32_MAX),
1344                 .index = 0xff,
1345         };
1346         int ret;
1347
1348         if (!mlx5_flow_ext_mreg_supported(dev))
1349                 return rte_flow_error_set(error, ENOTSUP,
1350                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1351                                           "extensive metadata register"
1352                                           " isn't supported");
1353         if (!spec)
1354                 return rte_flow_error_set(error, EINVAL,
1355                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1356                                           item->spec,
1357                                           "data cannot be empty");
1358         if (!mask)
1359                 mask = &rte_flow_item_tag_mask;
1360         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1361                                         (const uint8_t *)&nic_mask,
1362                                         sizeof(struct rte_flow_item_tag),
1363                                         error);
1364         if (ret < 0)
1365                 return ret;
1366         if (mask->index != 0xff)
1367                 return rte_flow_error_set(error, EINVAL,
1368                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1369                                           "partial mask for tag index"
1370                                           " is not supported");
1371         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1372         if (ret < 0)
1373                 return ret;
1374         assert(ret != REG_NONE);
1375         return 0;
1376 }
1377
1378 /**
1379  * Validate vport item.
1380  *
1381  * @param[in] dev
1382  *   Pointer to the rte_eth_dev structure.
1383  * @param[in] item
1384  *   Item specification.
1385  * @param[in] attr
1386  *   Attributes of flow that includes this item.
1387  * @param[in] item_flags
1388  *   Bit-fields that holds the items detected until now.
1389  * @param[out] error
1390  *   Pointer to error structure.
1391  *
1392  * @return
1393  *   0 on success, a negative errno value otherwise and rte_errno is set.
1394  */
1395 static int
1396 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1397                               const struct rte_flow_item *item,
1398                               const struct rte_flow_attr *attr,
1399                               uint64_t item_flags,
1400                               struct rte_flow_error *error)
1401 {
1402         const struct rte_flow_item_port_id *spec = item->spec;
1403         const struct rte_flow_item_port_id *mask = item->mask;
1404         const struct rte_flow_item_port_id switch_mask = {
1405                         .id = 0xffffffff,
1406         };
1407         struct mlx5_priv *esw_priv;
1408         struct mlx5_priv *dev_priv;
1409         int ret;
1410
1411         if (!attr->transfer)
1412                 return rte_flow_error_set(error, EINVAL,
1413                                           RTE_FLOW_ERROR_TYPE_ITEM,
1414                                           NULL,
1415                                           "match on port id is valid only"
1416                                           " when transfer flag is enabled");
1417         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1418                 return rte_flow_error_set(error, ENOTSUP,
1419                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1420                                           "multiple source ports are not"
1421                                           " supported");
1422         if (!mask)
1423                 mask = &switch_mask;
1424         if (mask->id != 0xffffffff)
1425                 return rte_flow_error_set(error, ENOTSUP,
1426                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1427                                            mask,
1428                                            "no support for partial mask on"
1429                                            " \"id\" field");
1430         ret = mlx5_flow_item_acceptable
1431                                 (item, (const uint8_t *)mask,
1432                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1433                                  sizeof(struct rte_flow_item_port_id),
1434                                  error);
1435         if (ret)
1436                 return ret;
1437         if (!spec)
1438                 return 0;
1439         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1440         if (!esw_priv)
1441                 return rte_flow_error_set(error, rte_errno,
1442                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1443                                           "failed to obtain E-Switch info for"
1444                                           " port");
1445         dev_priv = mlx5_dev_to_eswitch_info(dev);
1446         if (!dev_priv)
1447                 return rte_flow_error_set(error, rte_errno,
1448                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1449                                           NULL,
1450                                           "failed to obtain E-Switch info");
1451         if (esw_priv->domain_id != dev_priv->domain_id)
1452                 return rte_flow_error_set(error, EINVAL,
1453                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1454                                           "cannot match on a port from a"
1455                                           " different E-Switch");
1456         return 0;
1457 }
1458
1459 /**
1460  * Validate the pop VLAN action.
1461  *
1462  * @param[in] dev
1463  *   Pointer to the rte_eth_dev structure.
1464  * @param[in] action_flags
1465  *   Holds the actions detected until now.
1466  * @param[in] action
1467  *   Pointer to the pop vlan action.
1468  * @param[in] item_flags
1469  *   The items found in this flow rule.
1470  * @param[in] attr
1471  *   Pointer to flow attributes.
1472  * @param[out] error
1473  *   Pointer to error structure.
1474  *
1475  * @return
1476  *   0 on success, a negative errno value otherwise and rte_errno is set.
1477  */
1478 static int
1479 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1480                                  uint64_t action_flags,
1481                                  const struct rte_flow_action *action,
1482                                  uint64_t item_flags,
1483                                  const struct rte_flow_attr *attr,
1484                                  struct rte_flow_error *error)
1485 {
1486         struct mlx5_priv *priv = dev->data->dev_private;
1487
1488         (void)action;
1489         (void)attr;
1490         if (!priv->sh->pop_vlan_action)
1491                 return rte_flow_error_set(error, ENOTSUP,
1492                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1493                                           NULL,
1494                                           "pop vlan action is not supported");
1495         /*
1496          * Check for inconsistencies:
1497          *  fail strip_vlan in a flow that matches packets without VLAN tags.
1498          *  fail strip_vlan in a flow that matches packets without explicitly a
1499          *  matching on VLAN tag ?
1500          */
1501         if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN)
1502                 return rte_flow_error_set(error, ENOTSUP,
1503                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1504                                           NULL,
1505                                           "no support for multiple vlan pop "
1506                                           "actions");
1507         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1508                 return rte_flow_error_set(error, ENOTSUP,
1509                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1510                                           NULL,
1511                                           "cannot pop vlan without a "
1512                                           "match on (outer) vlan in the flow");
1513         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1514                 return rte_flow_error_set(error, EINVAL,
1515                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1516                                           "wrong action order, port_id should "
1517                                           "be after pop VLAN action");
1518         return 0;
1519 }
1520
1521 /**
1522  * Get VLAN default info from vlan match info.
1523  *
1524  * @param[in] dev
1525  *   Pointer to the rte_eth_dev structure.
1526  * @param[in] item
1527  *   the list of item specifications.
1528  * @param[out] vlan
1529  *   pointer VLAN info to fill to.
1530  * @param[out] error
1531  *   Pointer to error structure.
1532  *
1533  * @return
1534  *   0 on success, a negative errno value otherwise and rte_errno is set.
1535  */
1536 static void
1537 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1538                                   struct rte_vlan_hdr *vlan)
1539 {
1540         const struct rte_flow_item_vlan nic_mask = {
1541                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1542                                 MLX5DV_FLOW_VLAN_VID_MASK),
1543                 .inner_type = RTE_BE16(0xffff),
1544         };
1545
1546         if (items == NULL)
1547                 return;
1548         for (; items->type != RTE_FLOW_ITEM_TYPE_END &&
1549                items->type != RTE_FLOW_ITEM_TYPE_VLAN; items++)
1550                 ;
1551         if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1552                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1553                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1554
1555                 if (!vlan_m)
1556                         vlan_m = &nic_mask;
1557                 /* Only full match values are accepted */
1558                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1559                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1560                         vlan->vlan_tci &= MLX5DV_FLOW_VLAN_PCP_MASK;
1561                         vlan->vlan_tci |=
1562                                 rte_be_to_cpu_16(vlan_v->tci &
1563                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1564                 }
1565                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1566                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1567                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1568                         vlan->vlan_tci |=
1569                                 rte_be_to_cpu_16(vlan_v->tci &
1570                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1571                 }
1572                 if (vlan_m->inner_type == nic_mask.inner_type)
1573                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1574                                                            vlan_m->inner_type);
1575         }
1576 }
1577
1578 /**
1579  * Validate the push VLAN action.
1580  *
1581  * @param[in] action_flags
1582  *   Holds the actions detected until now.
1583  * @param[in] action
1584  *   Pointer to the encap action.
1585  * @param[in] attr
1586  *   Pointer to flow attributes
1587  * @param[out] error
1588  *   Pointer to error structure.
1589  *
1590  * @return
1591  *   0 on success, a negative errno value otherwise and rte_errno is set.
1592  */
1593 static int
1594 flow_dv_validate_action_push_vlan(uint64_t action_flags,
1595                                   uint64_t item_flags,
1596                                   const struct rte_flow_action *action,
1597                                   const struct rte_flow_attr *attr,
1598                                   struct rte_flow_error *error)
1599 {
1600         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1601
1602         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1603             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1604                 return rte_flow_error_set(error, EINVAL,
1605                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1606                                           "invalid vlan ethertype");
1607         if (action_flags &
1608                 (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1609                 return rte_flow_error_set(error, ENOTSUP,
1610                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1611                                           "no support for multiple VLAN "
1612                                           "actions");
1613         if (!mlx5_flow_find_action
1614                         (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) &&
1615             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1616                 return rte_flow_error_set(error, ENOTSUP,
1617                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
1618                                 "push VLAN needs to match on VLAN in order to "
1619                                 "get VLAN VID information because there is "
1620                                 "no followed set VLAN VID action");
1621         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1622                 return rte_flow_error_set(error, EINVAL,
1623                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1624                                           "wrong action order, port_id should "
1625                                           "be after push VLAN");
1626         (void)attr;
1627         return 0;
1628 }
1629
1630 /**
1631  * Validate the set VLAN PCP.
1632  *
1633  * @param[in] action_flags
1634  *   Holds the actions detected until now.
1635  * @param[in] actions
1636  *   Pointer to the list of actions remaining in the flow rule.
1637  * @param[in] attr
1638  *   Pointer to flow attributes
1639  * @param[out] error
1640  *   Pointer to error structure.
1641  *
1642  * @return
1643  *   0 on success, a negative errno value otherwise and rte_errno is set.
1644  */
1645 static int
1646 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
1647                                      const struct rte_flow_action actions[],
1648                                      struct rte_flow_error *error)
1649 {
1650         const struct rte_flow_action *action = actions;
1651         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
1652
1653         if (conf->vlan_pcp > 7)
1654                 return rte_flow_error_set(error, EINVAL,
1655                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1656                                           "VLAN PCP value is too big");
1657         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1658                 return rte_flow_error_set(error, ENOTSUP,
1659                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1660                                           "set VLAN PCP action must follow "
1661                                           "the push VLAN action");
1662         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
1663                 return rte_flow_error_set(error, ENOTSUP,
1664                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1665                                           "Multiple VLAN PCP modification are "
1666                                           "not supported");
1667         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1668                 return rte_flow_error_set(error, EINVAL,
1669                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1670                                           "wrong action order, port_id should "
1671                                           "be after set VLAN PCP");
1672         return 0;
1673 }
1674
1675 /**
1676  * Validate the set VLAN VID.
1677  *
1678  * @param[in] item_flags
1679  *   Holds the items detected in this rule.
1680  * @param[in] actions
1681  *   Pointer to the list of actions remaining in the flow rule.
1682  * @param[in] attr
1683  *   Pointer to flow attributes
1684  * @param[out] error
1685  *   Pointer to error structure.
1686  *
1687  * @return
1688  *   0 on success, a negative errno value otherwise and rte_errno is set.
1689  */
1690 static int
1691 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
1692                                      uint64_t action_flags,
1693                                      const struct rte_flow_action actions[],
1694                                      struct rte_flow_error *error)
1695 {
1696         const struct rte_flow_action *action = actions;
1697         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
1698
1699         if (conf->vlan_vid > RTE_BE16(0xFFE))
1700                 return rte_flow_error_set(error, EINVAL,
1701                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1702                                           "VLAN VID value is too big");
1703         /* there is an of_push_vlan action before us */
1704         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) {
1705                 if (mlx5_flow_find_action(actions + 1,
1706                                           RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID))
1707                         return rte_flow_error_set(error, ENOTSUP,
1708                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1709                                         "Multiple VLAN VID modifications are "
1710                                         "not supported");
1711                 else
1712                         return 0;
1713         }
1714
1715         /*
1716          * Action is on an existing VLAN header:
1717          *    Need to verify this is a single modify CID action.
1718          *   Rule mast include a match on outer VLAN.
1719          */
1720         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
1721                 return rte_flow_error_set(error, ENOTSUP,
1722                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1723                                           "Multiple VLAN VID modifications are "
1724                                           "not supported");
1725         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1726                 return rte_flow_error_set(error, EINVAL,
1727                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1728                                           "match on VLAN is required in order "
1729                                           "to set VLAN VID");
1730         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1731                 return rte_flow_error_set(error, EINVAL,
1732                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1733                                           "wrong action order, port_id should "
1734                                           "be after set VLAN VID");
1735         return 0;
1736 }
1737
1738 /*
1739  * Validate the FLAG action.
1740  *
1741  * @param[in] dev
1742  *   Pointer to the rte_eth_dev structure.
1743  * @param[in] action_flags
1744  *   Holds the actions detected until now.
1745  * @param[in] attr
1746  *   Pointer to flow attributes
1747  * @param[out] error
1748  *   Pointer to error structure.
1749  *
1750  * @return
1751  *   0 on success, a negative errno value otherwise and rte_errno is set.
1752  */
1753 static int
1754 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
1755                              uint64_t action_flags,
1756                              const struct rte_flow_attr *attr,
1757                              struct rte_flow_error *error)
1758 {
1759         struct mlx5_priv *priv = dev->data->dev_private;
1760         struct mlx5_dev_config *config = &priv->config;
1761         int ret;
1762
1763         /* Fall back if no extended metadata register support. */
1764         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1765                 return mlx5_flow_validate_action_flag(action_flags, attr,
1766                                                       error);
1767         /* Extensive metadata mode requires registers. */
1768         if (!mlx5_flow_ext_mreg_supported(dev))
1769                 return rte_flow_error_set(error, ENOTSUP,
1770                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1771                                           "no metadata registers "
1772                                           "to support flag action");
1773         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1776                                           "extended metadata register"
1777                                           " isn't available");
1778         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1779         if (ret < 0)
1780                 return ret;
1781         assert(ret > 0);
1782         if (action_flags & MLX5_FLOW_ACTION_DROP)
1783                 return rte_flow_error_set(error, EINVAL,
1784                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1785                                           "can't drop and flag in same flow");
1786         if (action_flags & MLX5_FLOW_ACTION_MARK)
1787                 return rte_flow_error_set(error, EINVAL,
1788                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1789                                           "can't mark and flag in same flow");
1790         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1791                 return rte_flow_error_set(error, EINVAL,
1792                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1793                                           "can't have 2 flag"
1794                                           " actions in same flow");
1795         return 0;
1796 }
1797
1798 /**
1799  * Validate MARK action.
1800  *
1801  * @param[in] dev
1802  *   Pointer to the rte_eth_dev structure.
1803  * @param[in] action
1804  *   Pointer to action.
1805  * @param[in] action_flags
1806  *   Holds the actions detected until now.
1807  * @param[in] attr
1808  *   Pointer to flow attributes
1809  * @param[out] error
1810  *   Pointer to error structure.
1811  *
1812  * @return
1813  *   0 on success, a negative errno value otherwise and rte_errno is set.
1814  */
1815 static int
1816 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
1817                              const struct rte_flow_action *action,
1818                              uint64_t action_flags,
1819                              const struct rte_flow_attr *attr,
1820                              struct rte_flow_error *error)
1821 {
1822         struct mlx5_priv *priv = dev->data->dev_private;
1823         struct mlx5_dev_config *config = &priv->config;
1824         const struct rte_flow_action_mark *mark = action->conf;
1825         int ret;
1826
1827         /* Fall back if no extended metadata register support. */
1828         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1829                 return mlx5_flow_validate_action_mark(action, action_flags,
1830                                                       attr, error);
1831         /* Extensive metadata mode requires registers. */
1832         if (!mlx5_flow_ext_mreg_supported(dev))
1833                 return rte_flow_error_set(error, ENOTSUP,
1834                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1835                                           "no metadata registers "
1836                                           "to support mark action");
1837         if (!priv->sh->dv_mark_mask)
1838                 return rte_flow_error_set(error, ENOTSUP,
1839                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1840                                           "extended metadata register"
1841                                           " isn't available");
1842         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1843         if (ret < 0)
1844                 return ret;
1845         assert(ret > 0);
1846         if (!mark)
1847                 return rte_flow_error_set(error, EINVAL,
1848                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1849                                           "configuration cannot be null");
1850         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
1851                 return rte_flow_error_set(error, EINVAL,
1852                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1853                                           &mark->id,
1854                                           "mark id exceeds the limit");
1855         if (action_flags & MLX5_FLOW_ACTION_DROP)
1856                 return rte_flow_error_set(error, EINVAL,
1857                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1858                                           "can't drop and mark in same flow");
1859         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1860                 return rte_flow_error_set(error, EINVAL,
1861                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1862                                           "can't flag and mark in same flow");
1863         if (action_flags & MLX5_FLOW_ACTION_MARK)
1864                 return rte_flow_error_set(error, EINVAL,
1865                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1866                                           "can't have 2 mark actions in same"
1867                                           " flow");
1868         return 0;
1869 }
1870
1871 /**
1872  * Validate SET_META action.
1873  *
1874  * @param[in] dev
1875  *   Pointer to the rte_eth_dev structure.
1876  * @param[in] action
1877  *   Pointer to the encap action.
1878  * @param[in] action_flags
1879  *   Holds the actions detected until now.
1880  * @param[in] attr
1881  *   Pointer to flow attributes
1882  * @param[out] error
1883  *   Pointer to error structure.
1884  *
1885  * @return
1886  *   0 on success, a negative errno value otherwise and rte_errno is set.
1887  */
1888 static int
1889 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
1890                                  const struct rte_flow_action *action,
1891                                  uint64_t action_flags __rte_unused,
1892                                  const struct rte_flow_attr *attr,
1893                                  struct rte_flow_error *error)
1894 {
1895         const struct rte_flow_action_set_meta *conf;
1896         uint32_t nic_mask = UINT32_MAX;
1897         enum modify_reg reg;
1898
1899         if (!mlx5_flow_ext_mreg_supported(dev))
1900                 return rte_flow_error_set(error, ENOTSUP,
1901                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1902                                           "extended metadata register"
1903                                           " isn't supported");
1904         reg = flow_dv_get_metadata_reg(dev, attr, error);
1905         if (reg < 0)
1906                 return reg;
1907         if (reg != REG_A && reg != REG_B) {
1908                 struct mlx5_priv *priv = dev->data->dev_private;
1909
1910                 nic_mask = priv->sh->dv_meta_mask;
1911         }
1912         if (!(action->conf))
1913                 return rte_flow_error_set(error, EINVAL,
1914                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1915                                           "configuration cannot be null");
1916         conf = (const struct rte_flow_action_set_meta *)action->conf;
1917         if (!conf->mask)
1918                 return rte_flow_error_set(error, EINVAL,
1919                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1920                                           "zero mask doesn't have any effect");
1921         if (conf->mask & ~nic_mask)
1922                 return rte_flow_error_set(error, EINVAL,
1923                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1924                                           "meta data must be within reg C0");
1925         if (!(conf->data & conf->mask))
1926                 return rte_flow_error_set(error, EINVAL,
1927                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1928                                           "zero value has no effect");
1929         return 0;
1930 }
1931
1932 /**
1933  * Validate SET_TAG action.
1934  *
1935  * @param[in] dev
1936  *   Pointer to the rte_eth_dev structure.
1937  * @param[in] action
1938  *   Pointer to the encap action.
1939  * @param[in] action_flags
1940  *   Holds the actions detected until now.
1941  * @param[in] attr
1942  *   Pointer to flow attributes
1943  * @param[out] error
1944  *   Pointer to error structure.
1945  *
1946  * @return
1947  *   0 on success, a negative errno value otherwise and rte_errno is set.
1948  */
1949 static int
1950 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
1951                                 const struct rte_flow_action *action,
1952                                 uint64_t action_flags,
1953                                 const struct rte_flow_attr *attr,
1954                                 struct rte_flow_error *error)
1955 {
1956         const struct rte_flow_action_set_tag *conf;
1957         const uint64_t terminal_action_flags =
1958                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
1959                 MLX5_FLOW_ACTION_RSS;
1960         int ret;
1961
1962         if (!mlx5_flow_ext_mreg_supported(dev))
1963                 return rte_flow_error_set(error, ENOTSUP,
1964                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1965                                           "extensive metadata register"
1966                                           " isn't supported");
1967         if (!(action->conf))
1968                 return rte_flow_error_set(error, EINVAL,
1969                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1970                                           "configuration cannot be null");
1971         conf = (const struct rte_flow_action_set_tag *)action->conf;
1972         if (!conf->mask)
1973                 return rte_flow_error_set(error, EINVAL,
1974                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1975                                           "zero mask doesn't have any effect");
1976         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1977         if (ret < 0)
1978                 return ret;
1979         if (!attr->transfer && attr->ingress &&
1980             (action_flags & terminal_action_flags))
1981                 return rte_flow_error_set(error, EINVAL,
1982                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1983                                           "set_tag has no effect"
1984                                           " with terminal actions");
1985         return 0;
1986 }
1987
1988 /**
1989  * Validate count action.
1990  *
1991  * @param[in] dev
1992  *   device otr.
1993  * @param[out] error
1994  *   Pointer to error structure.
1995  *
1996  * @return
1997  *   0 on success, a negative errno value otherwise and rte_errno is set.
1998  */
1999 static int
2000 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2001                               struct rte_flow_error *error)
2002 {
2003         struct mlx5_priv *priv = dev->data->dev_private;
2004
2005         if (!priv->config.devx)
2006                 goto notsup_err;
2007 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2008         return 0;
2009 #endif
2010 notsup_err:
2011         return rte_flow_error_set
2012                       (error, ENOTSUP,
2013                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2014                        NULL,
2015                        "count action not supported");
2016 }
2017
2018 /**
2019  * Validate the L2 encap action.
2020  *
2021  * @param[in] action_flags
2022  *   Holds the actions detected until now.
2023  * @param[in] action
2024  *   Pointer to the encap action.
2025  * @param[in] attr
2026  *   Pointer to flow attributes
2027  * @param[out] error
2028  *   Pointer to error structure.
2029  *
2030  * @return
2031  *   0 on success, a negative errno value otherwise and rte_errno is set.
2032  */
2033 static int
2034 flow_dv_validate_action_l2_encap(uint64_t action_flags,
2035                                  const struct rte_flow_action *action,
2036                                  const struct rte_flow_attr *attr,
2037                                  struct rte_flow_error *error)
2038 {
2039         if (!(action->conf))
2040                 return rte_flow_error_set(error, EINVAL,
2041                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2042                                           "configuration cannot be null");
2043         if (action_flags & MLX5_FLOW_ACTION_DROP)
2044                 return rte_flow_error_set(error, EINVAL,
2045                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2046                                           "can't drop and encap in same flow");
2047         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
2048                 return rte_flow_error_set(error, EINVAL,
2049                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2050                                           "can only have a single encap or"
2051                                           " decap action in a flow");
2052         if (!attr->transfer && attr->ingress)
2053                 return rte_flow_error_set(error, ENOTSUP,
2054                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2055                                           NULL,
2056                                           "encap action not supported for "
2057                                           "ingress");
2058         return 0;
2059 }
2060
2061 /**
2062  * Validate the L2 decap action.
2063  *
2064  * @param[in] action_flags
2065  *   Holds the actions detected until now.
2066  * @param[in] attr
2067  *   Pointer to flow attributes
2068  * @param[out] error
2069  *   Pointer to error structure.
2070  *
2071  * @return
2072  *   0 on success, a negative errno value otherwise and rte_errno is set.
2073  */
2074 static int
2075 flow_dv_validate_action_l2_decap(uint64_t action_flags,
2076                                  const struct rte_flow_attr *attr,
2077                                  struct rte_flow_error *error)
2078 {
2079         if (action_flags & MLX5_FLOW_ACTION_DROP)
2080                 return rte_flow_error_set(error, EINVAL,
2081                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2082                                           "can't drop and decap in same flow");
2083         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
2084                 return rte_flow_error_set(error, EINVAL,
2085                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2086                                           "can only have a single encap or"
2087                                           " decap action in a flow");
2088         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2091                                           "can't have decap action after"
2092                                           " modify action");
2093         if (attr->egress)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2096                                           NULL,
2097                                           "decap action not supported for "
2098                                           "egress");
2099         return 0;
2100 }
2101
2102 /**
2103  * Validate the raw encap action.
2104  *
2105  * @param[in] action_flags
2106  *   Holds the actions detected until now.
2107  * @param[in] action
2108  *   Pointer to the encap action.
2109  * @param[in] attr
2110  *   Pointer to flow attributes
2111  * @param[out] error
2112  *   Pointer to error structure.
2113  *
2114  * @return
2115  *   0 on success, a negative errno value otherwise and rte_errno is set.
2116  */
2117 static int
2118 flow_dv_validate_action_raw_encap(uint64_t action_flags,
2119                                   const struct rte_flow_action *action,
2120                                   const struct rte_flow_attr *attr,
2121                                   struct rte_flow_error *error)
2122 {
2123         const struct rte_flow_action_raw_encap *raw_encap =
2124                 (const struct rte_flow_action_raw_encap *)action->conf;
2125         if (!(action->conf))
2126                 return rte_flow_error_set(error, EINVAL,
2127                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2128                                           "configuration cannot be null");
2129         if (action_flags & MLX5_FLOW_ACTION_DROP)
2130                 return rte_flow_error_set(error, EINVAL,
2131                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2132                                           "can't drop and encap in same flow");
2133         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
2134                 return rte_flow_error_set(error, EINVAL,
2135                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2136                                           "can only have a single encap"
2137                                           " action in a flow");
2138         /* encap without preceding decap is not supported for ingress */
2139         if (!attr->transfer &&  attr->ingress &&
2140             !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
2141                 return rte_flow_error_set(error, ENOTSUP,
2142                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2143                                           NULL,
2144                                           "encap action not supported for "
2145                                           "ingress");
2146         if (!raw_encap->size || !raw_encap->data)
2147                 return rte_flow_error_set(error, EINVAL,
2148                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2149                                           "raw encap data cannot be empty");
2150         return 0;
2151 }
2152
2153 /**
2154  * Validate the raw decap action.
2155  *
2156  * @param[in] action_flags
2157  *   Holds the actions detected until now.
2158  * @param[in] action
2159  *   Pointer to the encap action.
2160  * @param[in] attr
2161  *   Pointer to flow attributes
2162  * @param[out] error
2163  *   Pointer to error structure.
2164  *
2165  * @return
2166  *   0 on success, a negative errno value otherwise and rte_errno is set.
2167  */
2168 static int
2169 flow_dv_validate_action_raw_decap(uint64_t action_flags,
2170                                   const struct rte_flow_action *action,
2171                                   const struct rte_flow_attr *attr,
2172                                   struct rte_flow_error *error)
2173 {
2174         if (action_flags & MLX5_FLOW_ACTION_DROP)
2175                 return rte_flow_error_set(error, EINVAL,
2176                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2177                                           "can't drop and decap in same flow");
2178         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
2179                 return rte_flow_error_set(error, EINVAL,
2180                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2181                                           "can't have encap action before"
2182                                           " decap action");
2183         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
2184                 return rte_flow_error_set(error, EINVAL,
2185                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2186                                           "can only have a single decap"
2187                                           " action in a flow");
2188         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2189                 return rte_flow_error_set(error, EINVAL,
2190                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2191                                           "can't have decap action after"
2192                                           " modify action");
2193         /* decap action is valid on egress only if it is followed by encap */
2194         if (attr->egress) {
2195                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
2196                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
2197                        action++) {
2198                 }
2199                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
2200                         return rte_flow_error_set
2201                                         (error, ENOTSUP,
2202                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2203                                          NULL, "decap action not supported"
2204                                          " for egress");
2205         }
2206         return 0;
2207 }
2208
2209 /**
2210  * Find existing encap/decap resource or create and register a new one.
2211  *
2212  * @param[in, out] dev
2213  *   Pointer to rte_eth_dev structure.
2214  * @param[in, out] resource
2215  *   Pointer to encap/decap resource.
2216  * @parm[in, out] dev_flow
2217  *   Pointer to the dev_flow.
2218  * @param[out] error
2219  *   pointer to error structure.
2220  *
2221  * @return
2222  *   0 on success otherwise -errno and errno is set.
2223  */
2224 static int
2225 flow_dv_encap_decap_resource_register
2226                         (struct rte_eth_dev *dev,
2227                          struct mlx5_flow_dv_encap_decap_resource *resource,
2228                          struct mlx5_flow *dev_flow,
2229                          struct rte_flow_error *error)
2230 {
2231         struct mlx5_priv *priv = dev->data->dev_private;
2232         struct mlx5_ibv_shared *sh = priv->sh;
2233         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2234         struct mlx5dv_dr_domain *domain;
2235
2236         resource->flags = dev_flow->group ? 0 : 1;
2237         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2238                 domain = sh->fdb_domain;
2239         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2240                 domain = sh->rx_domain;
2241         else
2242                 domain = sh->tx_domain;
2243
2244         /* Lookup a matching resource from cache. */
2245         LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
2246                 if (resource->reformat_type == cache_resource->reformat_type &&
2247                     resource->ft_type == cache_resource->ft_type &&
2248                     resource->flags == cache_resource->flags &&
2249                     resource->size == cache_resource->size &&
2250                     !memcmp((const void *)resource->buf,
2251                             (const void *)cache_resource->buf,
2252                             resource->size)) {
2253                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2254                                 (void *)cache_resource,
2255                                 rte_atomic32_read(&cache_resource->refcnt));
2256                         rte_atomic32_inc(&cache_resource->refcnt);
2257                         dev_flow->dv.encap_decap = cache_resource;
2258                         return 0;
2259                 }
2260         }
2261         /* Register new encap/decap resource. */
2262         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2263         if (!cache_resource)
2264                 return rte_flow_error_set(error, ENOMEM,
2265                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2266                                           "cannot allocate resource memory");
2267         *cache_resource = *resource;
2268         cache_resource->verbs_action =
2269                 mlx5_glue->dv_create_flow_action_packet_reformat
2270                         (sh->ctx, cache_resource->reformat_type,
2271                          cache_resource->ft_type, domain, cache_resource->flags,
2272                          cache_resource->size,
2273                          (cache_resource->size ? cache_resource->buf : NULL));
2274         if (!cache_resource->verbs_action) {
2275                 rte_free(cache_resource);
2276                 return rte_flow_error_set(error, ENOMEM,
2277                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2278                                           NULL, "cannot create action");
2279         }
2280         rte_atomic32_init(&cache_resource->refcnt);
2281         rte_atomic32_inc(&cache_resource->refcnt);
2282         LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
2283         dev_flow->dv.encap_decap = cache_resource;
2284         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2285                 (void *)cache_resource,
2286                 rte_atomic32_read(&cache_resource->refcnt));
2287         return 0;
2288 }
2289
2290 /**
2291  * Find existing table jump resource or create and register a new one.
2292  *
2293  * @param[in, out] dev
2294  *   Pointer to rte_eth_dev structure.
2295  * @param[in, out] tbl
2296  *   Pointer to flow table resource.
2297  * @parm[in, out] dev_flow
2298  *   Pointer to the dev_flow.
2299  * @param[out] error
2300  *   pointer to error structure.
2301  *
2302  * @return
2303  *   0 on success otherwise -errno and errno is set.
2304  */
2305 static int
2306 flow_dv_jump_tbl_resource_register
2307                         (struct rte_eth_dev *dev __rte_unused,
2308                          struct mlx5_flow_tbl_resource *tbl,
2309                          struct mlx5_flow *dev_flow,
2310                          struct rte_flow_error *error)
2311 {
2312         struct mlx5_flow_tbl_data_entry *tbl_data =
2313                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2314         int cnt;
2315
2316         assert(tbl);
2317         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2318         if (!cnt) {
2319                 tbl_data->jump.action =
2320                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
2321                         (tbl->obj);
2322                 if (!tbl_data->jump.action)
2323                         return rte_flow_error_set(error, ENOMEM,
2324                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2325                                         NULL, "cannot create jump action");
2326                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2327                         (void *)&tbl_data->jump, cnt);
2328         } else {
2329                 assert(tbl_data->jump.action);
2330                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2331                         (void *)&tbl_data->jump, cnt);
2332         }
2333         rte_atomic32_inc(&tbl_data->jump.refcnt);
2334         dev_flow->dv.jump = &tbl_data->jump;
2335         return 0;
2336 }
2337
2338 /**
2339  * Find existing table port ID resource or create and register a new one.
2340  *
2341  * @param[in, out] dev
2342  *   Pointer to rte_eth_dev structure.
2343  * @param[in, out] resource
2344  *   Pointer to port ID action resource.
2345  * @parm[in, out] dev_flow
2346  *   Pointer to the dev_flow.
2347  * @param[out] error
2348  *   pointer to error structure.
2349  *
2350  * @return
2351  *   0 on success otherwise -errno and errno is set.
2352  */
2353 static int
2354 flow_dv_port_id_action_resource_register
2355                         (struct rte_eth_dev *dev,
2356                          struct mlx5_flow_dv_port_id_action_resource *resource,
2357                          struct mlx5_flow *dev_flow,
2358                          struct rte_flow_error *error)
2359 {
2360         struct mlx5_priv *priv = dev->data->dev_private;
2361         struct mlx5_ibv_shared *sh = priv->sh;
2362         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2363
2364         /* Lookup a matching resource from cache. */
2365         LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
2366                 if (resource->port_id == cache_resource->port_id) {
2367                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2368                                 "refcnt %d++",
2369                                 (void *)cache_resource,
2370                                 rte_atomic32_read(&cache_resource->refcnt));
2371                         rte_atomic32_inc(&cache_resource->refcnt);
2372                         dev_flow->dv.port_id_action = cache_resource;
2373                         return 0;
2374                 }
2375         }
2376         /* Register new port id action resource. */
2377         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2378         if (!cache_resource)
2379                 return rte_flow_error_set(error, ENOMEM,
2380                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2381                                           "cannot allocate resource memory");
2382         *cache_resource = *resource;
2383         /*
2384          * Depending on rdma_core version the glue routine calls
2385          * either mlx5dv_dr_action_create_dest_ib_port(domain, ibv_port)
2386          * or mlx5dv_dr_action_create_dest_vport(domain, vport_id).
2387          */
2388         cache_resource->action =
2389                 mlx5_glue->dr_create_flow_action_dest_port
2390                         (priv->sh->fdb_domain, resource->port_id);
2391         if (!cache_resource->action) {
2392                 rte_free(cache_resource);
2393                 return rte_flow_error_set(error, ENOMEM,
2394                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2395                                           NULL, "cannot create action");
2396         }
2397         rte_atomic32_init(&cache_resource->refcnt);
2398         rte_atomic32_inc(&cache_resource->refcnt);
2399         LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
2400         dev_flow->dv.port_id_action = cache_resource;
2401         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2402                 (void *)cache_resource,
2403                 rte_atomic32_read(&cache_resource->refcnt));
2404         return 0;
2405 }
2406
2407 /**
2408  * Find existing push vlan resource or create and register a new one.
2409  *
2410  * @param [in, out] dev
2411  *   Pointer to rte_eth_dev structure.
2412  * @param[in, out] resource
2413  *   Pointer to port ID action resource.
2414  * @parm[in, out] dev_flow
2415  *   Pointer to the dev_flow.
2416  * @param[out] error
2417  *   pointer to error structure.
2418  *
2419  * @return
2420  *   0 on success otherwise -errno and errno is set.
2421  */
2422 static int
2423 flow_dv_push_vlan_action_resource_register
2424                        (struct rte_eth_dev *dev,
2425                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2426                         struct mlx5_flow *dev_flow,
2427                         struct rte_flow_error *error)
2428 {
2429         struct mlx5_priv *priv = dev->data->dev_private;
2430         struct mlx5_ibv_shared *sh = priv->sh;
2431         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2432         struct mlx5dv_dr_domain *domain;
2433
2434         /* Lookup a matching resource from cache. */
2435         LIST_FOREACH(cache_resource, &sh->push_vlan_action_list, next) {
2436                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2437                     resource->ft_type == cache_resource->ft_type) {
2438                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2439                                 "refcnt %d++",
2440                                 (void *)cache_resource,
2441                                 rte_atomic32_read(&cache_resource->refcnt));
2442                         rte_atomic32_inc(&cache_resource->refcnt);
2443                         dev_flow->dv.push_vlan_res = cache_resource;
2444                         return 0;
2445                 }
2446         }
2447         /* Register new push_vlan action resource. */
2448         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2449         if (!cache_resource)
2450                 return rte_flow_error_set(error, ENOMEM,
2451                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2452                                           "cannot allocate resource memory");
2453         *cache_resource = *resource;
2454         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2455                 domain = sh->fdb_domain;
2456         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2457                 domain = sh->rx_domain;
2458         else
2459                 domain = sh->tx_domain;
2460         cache_resource->action =
2461                 mlx5_glue->dr_create_flow_action_push_vlan(domain,
2462                                                            resource->vlan_tag);
2463         if (!cache_resource->action) {
2464                 rte_free(cache_resource);
2465                 return rte_flow_error_set(error, ENOMEM,
2466                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2467                                           NULL, "cannot create action");
2468         }
2469         rte_atomic32_init(&cache_resource->refcnt);
2470         rte_atomic32_inc(&cache_resource->refcnt);
2471         LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next);
2472         dev_flow->dv.push_vlan_res = cache_resource;
2473         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2474                 (void *)cache_resource,
2475                 rte_atomic32_read(&cache_resource->refcnt));
2476         return 0;
2477 }
2478 /**
2479  * Get the size of specific rte_flow_item_type
2480  *
2481  * @param[in] item_type
2482  *   Tested rte_flow_item_type.
2483  *
2484  * @return
2485  *   sizeof struct item_type, 0 if void or irrelevant.
2486  */
2487 static size_t
2488 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2489 {
2490         size_t retval;
2491
2492         switch (item_type) {
2493         case RTE_FLOW_ITEM_TYPE_ETH:
2494                 retval = sizeof(struct rte_flow_item_eth);
2495                 break;
2496         case RTE_FLOW_ITEM_TYPE_VLAN:
2497                 retval = sizeof(struct rte_flow_item_vlan);
2498                 break;
2499         case RTE_FLOW_ITEM_TYPE_IPV4:
2500                 retval = sizeof(struct rte_flow_item_ipv4);
2501                 break;
2502         case RTE_FLOW_ITEM_TYPE_IPV6:
2503                 retval = sizeof(struct rte_flow_item_ipv6);
2504                 break;
2505         case RTE_FLOW_ITEM_TYPE_UDP:
2506                 retval = sizeof(struct rte_flow_item_udp);
2507                 break;
2508         case RTE_FLOW_ITEM_TYPE_TCP:
2509                 retval = sizeof(struct rte_flow_item_tcp);
2510                 break;
2511         case RTE_FLOW_ITEM_TYPE_VXLAN:
2512                 retval = sizeof(struct rte_flow_item_vxlan);
2513                 break;
2514         case RTE_FLOW_ITEM_TYPE_GRE:
2515                 retval = sizeof(struct rte_flow_item_gre);
2516                 break;
2517         case RTE_FLOW_ITEM_TYPE_NVGRE:
2518                 retval = sizeof(struct rte_flow_item_nvgre);
2519                 break;
2520         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2521                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2522                 break;
2523         case RTE_FLOW_ITEM_TYPE_MPLS:
2524                 retval = sizeof(struct rte_flow_item_mpls);
2525                 break;
2526         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2527         default:
2528                 retval = 0;
2529                 break;
2530         }
2531         return retval;
2532 }
2533
2534 #define MLX5_ENCAP_IPV4_VERSION         0x40
2535 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2536 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2537 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2538 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2539 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2540 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2541
2542 /**
2543  * Convert the encap action data from list of rte_flow_item to raw buffer
2544  *
2545  * @param[in] items
2546  *   Pointer to rte_flow_item objects list.
2547  * @param[out] buf
2548  *   Pointer to the output buffer.
2549  * @param[out] size
2550  *   Pointer to the output buffer size.
2551  * @param[out] error
2552  *   Pointer to the error structure.
2553  *
2554  * @return
2555  *   0 on success, a negative errno value otherwise and rte_errno is set.
2556  */
2557 static int
2558 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2559                            size_t *size, struct rte_flow_error *error)
2560 {
2561         struct rte_ether_hdr *eth = NULL;
2562         struct rte_vlan_hdr *vlan = NULL;
2563         struct rte_ipv4_hdr *ipv4 = NULL;
2564         struct rte_ipv6_hdr *ipv6 = NULL;
2565         struct rte_udp_hdr *udp = NULL;
2566         struct rte_vxlan_hdr *vxlan = NULL;
2567         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2568         struct rte_gre_hdr *gre = NULL;
2569         size_t len;
2570         size_t temp_size = 0;
2571
2572         if (!items)
2573                 return rte_flow_error_set(error, EINVAL,
2574                                           RTE_FLOW_ERROR_TYPE_ACTION,
2575                                           NULL, "invalid empty data");
2576         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2577                 len = flow_dv_get_item_len(items->type);
2578                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2579                         return rte_flow_error_set(error, EINVAL,
2580                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2581                                                   (void *)items->type,
2582                                                   "items total size is too big"
2583                                                   " for encap action");
2584                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2585                 switch (items->type) {
2586                 case RTE_FLOW_ITEM_TYPE_ETH:
2587                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2588                         break;
2589                 case RTE_FLOW_ITEM_TYPE_VLAN:
2590                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2591                         if (!eth)
2592                                 return rte_flow_error_set(error, EINVAL,
2593                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2594                                                 (void *)items->type,
2595                                                 "eth header not found");
2596                         if (!eth->ether_type)
2597                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2598                         break;
2599                 case RTE_FLOW_ITEM_TYPE_IPV4:
2600                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
2601                         if (!vlan && !eth)
2602                                 return rte_flow_error_set(error, EINVAL,
2603                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2604                                                 (void *)items->type,
2605                                                 "neither eth nor vlan"
2606                                                 " header found");
2607                         if (vlan && !vlan->eth_proto)
2608                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
2609                         else if (eth && !eth->ether_type)
2610                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
2611                         if (!ipv4->version_ihl)
2612                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
2613                                                     MLX5_ENCAP_IPV4_IHL_MIN;
2614                         if (!ipv4->time_to_live)
2615                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
2616                         break;
2617                 case RTE_FLOW_ITEM_TYPE_IPV6:
2618                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
2619                         if (!vlan && !eth)
2620                                 return rte_flow_error_set(error, EINVAL,
2621                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2622                                                 (void *)items->type,
2623                                                 "neither eth nor vlan"
2624                                                 " header found");
2625                         if (vlan && !vlan->eth_proto)
2626                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
2627                         else if (eth && !eth->ether_type)
2628                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
2629                         if (!ipv6->vtc_flow)
2630                                 ipv6->vtc_flow =
2631                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
2632                         if (!ipv6->hop_limits)
2633                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
2634                         break;
2635                 case RTE_FLOW_ITEM_TYPE_UDP:
2636                         udp = (struct rte_udp_hdr *)&buf[temp_size];
2637                         if (!ipv4 && !ipv6)
2638                                 return rte_flow_error_set(error, EINVAL,
2639                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2640                                                 (void *)items->type,
2641                                                 "ip header not found");
2642                         if (ipv4 && !ipv4->next_proto_id)
2643                                 ipv4->next_proto_id = IPPROTO_UDP;
2644                         else if (ipv6 && !ipv6->proto)
2645                                 ipv6->proto = IPPROTO_UDP;
2646                         break;
2647                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2648                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
2649                         if (!udp)
2650                                 return rte_flow_error_set(error, EINVAL,
2651                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2652                                                 (void *)items->type,
2653                                                 "udp header not found");
2654                         if (!udp->dst_port)
2655                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
2656                         if (!vxlan->vx_flags)
2657                                 vxlan->vx_flags =
2658                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
2659                         break;
2660                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2661                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
2662                         if (!udp)
2663                                 return rte_flow_error_set(error, EINVAL,
2664                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2665                                                 (void *)items->type,
2666                                                 "udp header not found");
2667                         if (!vxlan_gpe->proto)
2668                                 return rte_flow_error_set(error, EINVAL,
2669                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2670                                                 (void *)items->type,
2671                                                 "next protocol not found");
2672                         if (!udp->dst_port)
2673                                 udp->dst_port =
2674                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
2675                         if (!vxlan_gpe->vx_flags)
2676                                 vxlan_gpe->vx_flags =
2677                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
2678                         break;
2679                 case RTE_FLOW_ITEM_TYPE_GRE:
2680                 case RTE_FLOW_ITEM_TYPE_NVGRE:
2681                         gre = (struct rte_gre_hdr *)&buf[temp_size];
2682                         if (!gre->proto)
2683                                 return rte_flow_error_set(error, EINVAL,
2684                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2685                                                 (void *)items->type,
2686                                                 "next protocol not found");
2687                         if (!ipv4 && !ipv6)
2688                                 return rte_flow_error_set(error, EINVAL,
2689                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2690                                                 (void *)items->type,
2691                                                 "ip header not found");
2692                         if (ipv4 && !ipv4->next_proto_id)
2693                                 ipv4->next_proto_id = IPPROTO_GRE;
2694                         else if (ipv6 && !ipv6->proto)
2695                                 ipv6->proto = IPPROTO_GRE;
2696                         break;
2697                 case RTE_FLOW_ITEM_TYPE_VOID:
2698                         break;
2699                 default:
2700                         return rte_flow_error_set(error, EINVAL,
2701                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2702                                                   (void *)items->type,
2703                                                   "unsupported item type");
2704                         break;
2705                 }
2706                 temp_size += len;
2707         }
2708         *size = temp_size;
2709         return 0;
2710 }
2711
2712 static int
2713 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
2714 {
2715         struct rte_ether_hdr *eth = NULL;
2716         struct rte_vlan_hdr *vlan = NULL;
2717         struct rte_ipv6_hdr *ipv6 = NULL;
2718         struct rte_udp_hdr *udp = NULL;
2719         char *next_hdr;
2720         uint16_t proto;
2721
2722         eth = (struct rte_ether_hdr *)data;
2723         next_hdr = (char *)(eth + 1);
2724         proto = RTE_BE16(eth->ether_type);
2725
2726         /* VLAN skipping */
2727         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
2728                 vlan = (struct rte_vlan_hdr *)next_hdr;
2729                 proto = RTE_BE16(vlan->eth_proto);
2730                 next_hdr += sizeof(struct rte_vlan_hdr);
2731         }
2732
2733         /* HW calculates IPv4 csum. no need to proceed */
2734         if (proto == RTE_ETHER_TYPE_IPV4)
2735                 return 0;
2736
2737         /* non IPv4/IPv6 header. not supported */
2738         if (proto != RTE_ETHER_TYPE_IPV6) {
2739                 return rte_flow_error_set(error, ENOTSUP,
2740                                           RTE_FLOW_ERROR_TYPE_ACTION,
2741                                           NULL, "Cannot offload non IPv4/IPv6");
2742         }
2743
2744         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
2745
2746         /* ignore non UDP */
2747         if (ipv6->proto != IPPROTO_UDP)
2748                 return 0;
2749
2750         udp = (struct rte_udp_hdr *)(ipv6 + 1);
2751         udp->dgram_cksum = 0;
2752
2753         return 0;
2754 }
2755
2756 /**
2757  * Convert L2 encap action to DV specification.
2758  *
2759  * @param[in] dev
2760  *   Pointer to rte_eth_dev structure.
2761  * @param[in] action
2762  *   Pointer to action structure.
2763  * @param[in, out] dev_flow
2764  *   Pointer to the mlx5_flow.
2765  * @param[in] transfer
2766  *   Mark if the flow is E-Switch flow.
2767  * @param[out] error
2768  *   Pointer to the error structure.
2769  *
2770  * @return
2771  *   0 on success, a negative errno value otherwise and rte_errno is set.
2772  */
2773 static int
2774 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
2775                                const struct rte_flow_action *action,
2776                                struct mlx5_flow *dev_flow,
2777                                uint8_t transfer,
2778                                struct rte_flow_error *error)
2779 {
2780         const struct rte_flow_item *encap_data;
2781         const struct rte_flow_action_raw_encap *raw_encap_data;
2782         struct mlx5_flow_dv_encap_decap_resource res = {
2783                 .reformat_type =
2784                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
2785                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
2786                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
2787         };
2788
2789         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
2790                 raw_encap_data =
2791                         (const struct rte_flow_action_raw_encap *)action->conf;
2792                 res.size = raw_encap_data->size;
2793                 memcpy(res.buf, raw_encap_data->data, res.size);
2794                 if (flow_dv_zero_encap_udp_csum(res.buf, error))
2795                         return -rte_errno;
2796         } else {
2797                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
2798                         encap_data =
2799                                 ((const struct rte_flow_action_vxlan_encap *)
2800                                                 action->conf)->definition;
2801                 else
2802                         encap_data =
2803                                 ((const struct rte_flow_action_nvgre_encap *)
2804                                                 action->conf)->definition;
2805                 if (flow_dv_convert_encap_data(encap_data, res.buf,
2806                                                &res.size, error))
2807                         return -rte_errno;
2808         }
2809         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
2810                 return rte_flow_error_set(error, EINVAL,
2811                                           RTE_FLOW_ERROR_TYPE_ACTION,
2812                                           NULL, "can't create L2 encap action");
2813         return 0;
2814 }
2815
2816 /**
2817  * Convert L2 decap action to DV specification.
2818  *
2819  * @param[in] dev
2820  *   Pointer to rte_eth_dev structure.
2821  * @param[in, out] dev_flow
2822  *   Pointer to the mlx5_flow.
2823  * @param[in] transfer
2824  *   Mark if the flow is E-Switch flow.
2825  * @param[out] error
2826  *   Pointer to the error structure.
2827  *
2828  * @return
2829  *   0 on success, a negative errno value otherwise and rte_errno is set.
2830  */
2831 static int
2832 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
2833                                struct mlx5_flow *dev_flow,
2834                                uint8_t transfer,
2835                                struct rte_flow_error *error)
2836 {
2837         struct mlx5_flow_dv_encap_decap_resource res = {
2838                 .size = 0,
2839                 .reformat_type =
2840                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
2841                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
2842                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
2843         };
2844
2845         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
2846                 return rte_flow_error_set(error, EINVAL,
2847                                           RTE_FLOW_ERROR_TYPE_ACTION,
2848                                           NULL, "can't create L2 decap action");
2849         return 0;
2850 }
2851
2852 /**
2853  * Convert raw decap/encap (L3 tunnel) action to DV specification.
2854  *
2855  * @param[in] dev
2856  *   Pointer to rte_eth_dev structure.
2857  * @param[in] action
2858  *   Pointer to action structure.
2859  * @param[in, out] dev_flow
2860  *   Pointer to the mlx5_flow.
2861  * @param[in] attr
2862  *   Pointer to the flow attributes.
2863  * @param[out] error
2864  *   Pointer to the error structure.
2865  *
2866  * @return
2867  *   0 on success, a negative errno value otherwise and rte_errno is set.
2868  */
2869 static int
2870 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
2871                                 const struct rte_flow_action *action,
2872                                 struct mlx5_flow *dev_flow,
2873                                 const struct rte_flow_attr *attr,
2874                                 struct rte_flow_error *error)
2875 {
2876         const struct rte_flow_action_raw_encap *encap_data;
2877         struct mlx5_flow_dv_encap_decap_resource res;
2878
2879         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
2880         res.size = encap_data->size;
2881         memcpy(res.buf, encap_data->data, res.size);
2882         res.reformat_type = attr->egress ?
2883                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
2884                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
2885         if (attr->transfer)
2886                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
2887         else
2888                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
2889                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
2890         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
2891                 return rte_flow_error_set(error, EINVAL,
2892                                           RTE_FLOW_ERROR_TYPE_ACTION,
2893                                           NULL, "can't create encap action");
2894         return 0;
2895 }
2896
2897 /**
2898  * Create action push VLAN.
2899  *
2900  * @param[in] dev
2901  *   Pointer to rte_eth_dev structure.
2902  * @param[in] vlan_tag
2903  *   the vlan tag to push to the Ethernet header.
2904  * @param[in, out] dev_flow
2905  *   Pointer to the mlx5_flow.
2906  * @param[in] attr
2907  *   Pointer to the flow attributes.
2908  * @param[out] error
2909  *   Pointer to the error structure.
2910  *
2911  * @return
2912  *   0 on success, a negative errno value otherwise and rte_errno is set.
2913  */
2914 static int
2915 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
2916                                 const struct rte_flow_attr *attr,
2917                                 const struct rte_vlan_hdr *vlan,
2918                                 struct mlx5_flow *dev_flow,
2919                                 struct rte_flow_error *error)
2920 {
2921         struct mlx5_flow_dv_push_vlan_action_resource res;
2922
2923         res.vlan_tag =
2924                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
2925                                  vlan->vlan_tci);
2926         if (attr->transfer)
2927                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
2928         else
2929                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
2930                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
2931         return flow_dv_push_vlan_action_resource_register
2932                                             (dev, &res, dev_flow, error);
2933 }
2934
2935 /**
2936  * Validate the modify-header actions.
2937  *
2938  * @param[in] action_flags
2939  *   Holds the actions detected until now.
2940  * @param[in] action
2941  *   Pointer to the modify action.
2942  * @param[out] error
2943  *   Pointer to error structure.
2944  *
2945  * @return
2946  *   0 on success, a negative errno value otherwise and rte_errno is set.
2947  */
2948 static int
2949 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
2950                                    const struct rte_flow_action *action,
2951                                    struct rte_flow_error *error)
2952 {
2953         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
2954                 return rte_flow_error_set(error, EINVAL,
2955                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2956                                           NULL, "action configuration not set");
2957         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
2958                 return rte_flow_error_set(error, EINVAL,
2959                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2960                                           "can't have encap action before"
2961                                           " modify action");
2962         return 0;
2963 }
2964
2965 /**
2966  * Validate the modify-header MAC address actions.
2967  *
2968  * @param[in] action_flags
2969  *   Holds the actions detected until now.
2970  * @param[in] action
2971  *   Pointer to the modify action.
2972  * @param[in] item_flags
2973  *   Holds the items detected.
2974  * @param[out] error
2975  *   Pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success, a negative errno value otherwise and rte_errno is set.
2979  */
2980 static int
2981 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
2982                                    const struct rte_flow_action *action,
2983                                    const uint64_t item_flags,
2984                                    struct rte_flow_error *error)
2985 {
2986         int ret = 0;
2987
2988         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2989         if (!ret) {
2990                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
2991                         return rte_flow_error_set(error, EINVAL,
2992                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2993                                                   NULL,
2994                                                   "no L2 item in pattern");
2995         }
2996         return ret;
2997 }
2998
2999 /**
3000  * Validate the modify-header IPv4 address actions.
3001  *
3002  * @param[in] action_flags
3003  *   Holds the actions detected until now.
3004  * @param[in] action
3005  *   Pointer to the modify action.
3006  * @param[in] item_flags
3007  *   Holds the items detected.
3008  * @param[out] error
3009  *   Pointer to error structure.
3010  *
3011  * @return
3012  *   0 on success, a negative errno value otherwise and rte_errno is set.
3013  */
3014 static int
3015 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3016                                     const struct rte_flow_action *action,
3017                                     const uint64_t item_flags,
3018                                     struct rte_flow_error *error)
3019 {
3020         int ret = 0;
3021
3022         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3023         if (!ret) {
3024                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3025                         return rte_flow_error_set(error, EINVAL,
3026                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3027                                                   NULL,
3028                                                   "no ipv4 item in pattern");
3029         }
3030         return ret;
3031 }
3032
3033 /**
3034  * Validate the modify-header IPv6 address actions.
3035  *
3036  * @param[in] action_flags
3037  *   Holds the actions detected until now.
3038  * @param[in] action
3039  *   Pointer to the modify action.
3040  * @param[in] item_flags
3041  *   Holds the items detected.
3042  * @param[out] error
3043  *   Pointer to error structure.
3044  *
3045  * @return
3046  *   0 on success, a negative errno value otherwise and rte_errno is set.
3047  */
3048 static int
3049 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3050                                     const struct rte_flow_action *action,
3051                                     const uint64_t item_flags,
3052                                     struct rte_flow_error *error)
3053 {
3054         int ret = 0;
3055
3056         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3057         if (!ret) {
3058                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3059                         return rte_flow_error_set(error, EINVAL,
3060                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3061                                                   NULL,
3062                                                   "no ipv6 item in pattern");
3063         }
3064         return ret;
3065 }
3066
3067 /**
3068  * Validate the modify-header TP actions.
3069  *
3070  * @param[in] action_flags
3071  *   Holds the actions detected until now.
3072  * @param[in] action
3073  *   Pointer to the modify action.
3074  * @param[in] item_flags
3075  *   Holds the items detected.
3076  * @param[out] error
3077  *   Pointer to error structure.
3078  *
3079  * @return
3080  *   0 on success, a negative errno value otherwise and rte_errno is set.
3081  */
3082 static int
3083 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3084                                   const struct rte_flow_action *action,
3085                                   const uint64_t item_flags,
3086                                   struct rte_flow_error *error)
3087 {
3088         int ret = 0;
3089
3090         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3091         if (!ret) {
3092                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
3093                         return rte_flow_error_set(error, EINVAL,
3094                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3095                                                   NULL, "no transport layer "
3096                                                   "in pattern");
3097         }
3098         return ret;
3099 }
3100
3101 /**
3102  * Validate the modify-header actions of increment/decrement
3103  * TCP Sequence-number.
3104  *
3105  * @param[in] action_flags
3106  *   Holds the actions detected until now.
3107  * @param[in] action
3108  *   Pointer to the modify action.
3109  * @param[in] item_flags
3110  *   Holds the items detected.
3111  * @param[out] error
3112  *   Pointer to error structure.
3113  *
3114  * @return
3115  *   0 on success, a negative errno value otherwise and rte_errno is set.
3116  */
3117 static int
3118 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3119                                        const struct rte_flow_action *action,
3120                                        const uint64_t item_flags,
3121                                        struct rte_flow_error *error)
3122 {
3123         int ret = 0;
3124
3125         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3126         if (!ret) {
3127                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3128                         return rte_flow_error_set(error, EINVAL,
3129                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3130                                                   NULL, "no TCP item in"
3131                                                   " pattern");
3132                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3133                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3134                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3135                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3136                         return rte_flow_error_set(error, EINVAL,
3137                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3138                                                   NULL,
3139                                                   "cannot decrease and increase"
3140                                                   " TCP sequence number"
3141                                                   " at the same time");
3142         }
3143         return ret;
3144 }
3145
3146 /**
3147  * Validate the modify-header actions of increment/decrement
3148  * TCP Acknowledgment number.
3149  *
3150  * @param[in] action_flags
3151  *   Holds the actions detected until now.
3152  * @param[in] action
3153  *   Pointer to the modify action.
3154  * @param[in] item_flags
3155  *   Holds the items detected.
3156  * @param[out] error
3157  *   Pointer to error structure.
3158  *
3159  * @return
3160  *   0 on success, a negative errno value otherwise and rte_errno is set.
3161  */
3162 static int
3163 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3164                                        const struct rte_flow_action *action,
3165                                        const uint64_t item_flags,
3166                                        struct rte_flow_error *error)
3167 {
3168         int ret = 0;
3169
3170         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3171         if (!ret) {
3172                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3173                         return rte_flow_error_set(error, EINVAL,
3174                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3175                                                   NULL, "no TCP item in"
3176                                                   " pattern");
3177                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3178                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3179                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3180                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3181                         return rte_flow_error_set(error, EINVAL,
3182                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3183                                                   NULL,
3184                                                   "cannot decrease and increase"
3185                                                   " TCP acknowledgment number"
3186                                                   " at the same time");
3187         }
3188         return ret;
3189 }
3190
3191 /**
3192  * Validate the modify-header TTL actions.
3193  *
3194  * @param[in] action_flags
3195  *   Holds the actions detected until now.
3196  * @param[in] action
3197  *   Pointer to the modify action.
3198  * @param[in] item_flags
3199  *   Holds the items detected.
3200  * @param[out] error
3201  *   Pointer to error structure.
3202  *
3203  * @return
3204  *   0 on success, a negative errno value otherwise and rte_errno is set.
3205  */
3206 static int
3207 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3208                                    const struct rte_flow_action *action,
3209                                    const uint64_t item_flags,
3210                                    struct rte_flow_error *error)
3211 {
3212         int ret = 0;
3213
3214         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3215         if (!ret) {
3216                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
3217                         return rte_flow_error_set(error, EINVAL,
3218                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3219                                                   NULL,
3220                                                   "no IP protocol in pattern");
3221         }
3222         return ret;
3223 }
3224
3225 /**
3226  * Validate jump action.
3227  *
3228  * @param[in] action
3229  *   Pointer to the jump action.
3230  * @param[in] action_flags
3231  *   Holds the actions detected until now.
3232  * @param[in] attributes
3233  *   Pointer to flow attributes
3234  * @param[in] external
3235  *   Action belongs to flow rule created by request external to PMD.
3236  * @param[out] error
3237  *   Pointer to error structure.
3238  *
3239  * @return
3240  *   0 on success, a negative errno value otherwise and rte_errno is set.
3241  */
3242 static int
3243 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3244                              uint64_t action_flags,
3245                              const struct rte_flow_attr *attributes,
3246                              bool external, struct rte_flow_error *error)
3247 {
3248         uint32_t target_group, table;
3249         int ret = 0;
3250
3251         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3252                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3253                 return rte_flow_error_set(error, EINVAL,
3254                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3255                                           "can't have 2 fate actions in"
3256                                           " same flow");
3257         if (action_flags & MLX5_FLOW_ACTION_METER)
3258                 return rte_flow_error_set(error, ENOTSUP,
3259                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3260                                           "jump with meter not support");
3261         if (!action->conf)
3262                 return rte_flow_error_set(error, EINVAL,
3263                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3264                                           NULL, "action configuration not set");
3265         target_group =
3266                 ((const struct rte_flow_action_jump *)action->conf)->group;
3267         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3268                                        &table, error);
3269         if (ret)
3270                 return ret;
3271         if (attributes->group == target_group)
3272                 return rte_flow_error_set(error, EINVAL,
3273                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3274                                           "target group must be other than"
3275                                           " the current flow group");
3276         return 0;
3277 }
3278
3279 /*
3280  * Validate the port_id action.
3281  *
3282  * @param[in] dev
3283  *   Pointer to rte_eth_dev structure.
3284  * @param[in] action_flags
3285  *   Bit-fields that holds the actions detected until now.
3286  * @param[in] action
3287  *   Port_id RTE action structure.
3288  * @param[in] attr
3289  *   Attributes of flow that includes this action.
3290  * @param[out] error
3291  *   Pointer to error structure.
3292  *
3293  * @return
3294  *   0 on success, a negative errno value otherwise and rte_errno is set.
3295  */
3296 static int
3297 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3298                                 uint64_t action_flags,
3299                                 const struct rte_flow_action *action,
3300                                 const struct rte_flow_attr *attr,
3301                                 struct rte_flow_error *error)
3302 {
3303         const struct rte_flow_action_port_id *port_id;
3304         struct mlx5_priv *act_priv;
3305         struct mlx5_priv *dev_priv;
3306         uint16_t port;
3307
3308         if (!attr->transfer)
3309                 return rte_flow_error_set(error, ENOTSUP,
3310                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3311                                           NULL,
3312                                           "port id action is valid in transfer"
3313                                           " mode only");
3314         if (!action || !action->conf)
3315                 return rte_flow_error_set(error, ENOTSUP,
3316                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3317                                           NULL,
3318                                           "port id action parameters must be"
3319                                           " specified");
3320         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3321                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3322                 return rte_flow_error_set(error, EINVAL,
3323                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3324                                           "can have only one fate actions in"
3325                                           " a flow");
3326         dev_priv = mlx5_dev_to_eswitch_info(dev);
3327         if (!dev_priv)
3328                 return rte_flow_error_set(error, rte_errno,
3329                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3330                                           NULL,
3331                                           "failed to obtain E-Switch info");
3332         port_id = action->conf;
3333         port = port_id->original ? dev->data->port_id : port_id->id;
3334         act_priv = mlx5_port_to_eswitch_info(port, false);
3335         if (!act_priv)
3336                 return rte_flow_error_set
3337                                 (error, rte_errno,
3338                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3339                                  "failed to obtain E-Switch port id for port");
3340         if (act_priv->domain_id != dev_priv->domain_id)
3341                 return rte_flow_error_set
3342                                 (error, EINVAL,
3343                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3344                                  "port does not belong to"
3345                                  " E-Switch being configured");
3346         return 0;
3347 }
3348
3349 /**
3350  * Get the maximum number of modify header actions.
3351  *
3352  * @param dev
3353  *   Pointer to rte_eth_dev structure.
3354  *
3355  * @return
3356  *   Max number of modify header actions device can support.
3357  */
3358 static unsigned int
3359 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev)
3360 {
3361         /*
3362          * There's no way to directly query the max cap. Although it has to be
3363          * acquried by iterative trial, it is a safe assumption that more
3364          * actions are supported by FW if extensive metadata register is
3365          * supported.
3366          */
3367         return mlx5_flow_ext_mreg_supported(dev) ? MLX5_MODIFY_NUM :
3368                                                    MLX5_MODIFY_NUM_NO_MREG;
3369 }
3370
3371 /**
3372  * Validate the meter action.
3373  *
3374  * @param[in] dev
3375  *   Pointer to rte_eth_dev structure.
3376  * @param[in] action_flags
3377  *   Bit-fields that holds the actions detected until now.
3378  * @param[in] action
3379  *   Pointer to the meter action.
3380  * @param[in] attr
3381  *   Attributes of flow that includes this action.
3382  * @param[out] error
3383  *   Pointer to error structure.
3384  *
3385  * @return
3386  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3387  */
3388 static int
3389 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3390                                 uint64_t action_flags,
3391                                 const struct rte_flow_action *action,
3392                                 const struct rte_flow_attr *attr,
3393                                 struct rte_flow_error *error)
3394 {
3395         struct mlx5_priv *priv = dev->data->dev_private;
3396         const struct rte_flow_action_meter *am = action->conf;
3397         struct mlx5_flow_meter *fm = mlx5_flow_meter_find(priv, am->mtr_id);
3398
3399         if (action_flags & MLX5_FLOW_ACTION_METER)
3400                 return rte_flow_error_set(error, ENOTSUP,
3401                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3402                                           "meter chaining not support");
3403         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3404                 return rte_flow_error_set(error, ENOTSUP,
3405                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3406                                           "meter with jump not support");
3407         if (!priv->mtr_en)
3408                 return rte_flow_error_set(error, ENOTSUP,
3409                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3410                                           NULL,
3411                                           "meter action not supported");
3412         if (!fm)
3413                 return rte_flow_error_set(error, EINVAL,
3414                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3415                                           "Meter not found");
3416         if (fm->ref_cnt && (!(fm->attr.transfer == attr->transfer ||
3417               (!fm->attr.ingress && !attr->ingress && attr->egress) ||
3418               (!fm->attr.egress && !attr->egress && attr->ingress))))
3419                 return rte_flow_error_set(error, EINVAL,
3420                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3421                                           "Flow attributes are either invalid "
3422                                           "or have a conflict with current "
3423                                           "meter attributes");
3424         return 0;
3425 }
3426
3427 /**
3428  * Find existing modify-header resource or create and register a new one.
3429  *
3430  * @param dev[in, out]
3431  *   Pointer to rte_eth_dev structure.
3432  * @param[in, out] resource
3433  *   Pointer to modify-header resource.
3434  * @parm[in, out] dev_flow
3435  *   Pointer to the dev_flow.
3436  * @param[out] error
3437  *   pointer to error structure.
3438  *
3439  * @return
3440  *   0 on success otherwise -errno and errno is set.
3441  */
3442 static int
3443 flow_dv_modify_hdr_resource_register
3444                         (struct rte_eth_dev *dev,
3445                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3446                          struct mlx5_flow *dev_flow,
3447                          struct rte_flow_error *error)
3448 {
3449         struct mlx5_priv *priv = dev->data->dev_private;
3450         struct mlx5_ibv_shared *sh = priv->sh;
3451         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3452         struct mlx5dv_dr_domain *ns;
3453
3454         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev))
3455                 return rte_flow_error_set(error, EOVERFLOW,
3456                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3457                                           "too many modify header items");
3458         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3459                 ns = sh->fdb_domain;
3460         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
3461                 ns = sh->tx_domain;
3462         else
3463                 ns = sh->rx_domain;
3464         resource->flags =
3465                 dev_flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
3466         /* Lookup a matching resource from cache. */
3467         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
3468                 if (resource->ft_type == cache_resource->ft_type &&
3469                     resource->actions_num == cache_resource->actions_num &&
3470                     resource->flags == cache_resource->flags &&
3471                     !memcmp((const void *)resource->actions,
3472                             (const void *)cache_resource->actions,
3473                             (resource->actions_num *
3474                                             sizeof(resource->actions[0])))) {
3475                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
3476                                 (void *)cache_resource,
3477                                 rte_atomic32_read(&cache_resource->refcnt));
3478                         rte_atomic32_inc(&cache_resource->refcnt);
3479                         dev_flow->dv.modify_hdr = cache_resource;
3480                         return 0;
3481                 }
3482         }
3483         /* Register new modify-header resource. */
3484         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
3485         if (!cache_resource)
3486                 return rte_flow_error_set(error, ENOMEM,
3487                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3488                                           "cannot allocate resource memory");
3489         *cache_resource = *resource;
3490         cache_resource->verbs_action =
3491                 mlx5_glue->dv_create_flow_action_modify_header
3492                                         (sh->ctx, cache_resource->ft_type,
3493                                          ns, cache_resource->flags,
3494                                          cache_resource->actions_num *
3495                                          sizeof(cache_resource->actions[0]),
3496                                          (uint64_t *)cache_resource->actions);
3497         if (!cache_resource->verbs_action) {
3498                 rte_free(cache_resource);
3499                 return rte_flow_error_set(error, ENOMEM,
3500                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3501                                           NULL, "cannot create action");
3502         }
3503         rte_atomic32_init(&cache_resource->refcnt);
3504         rte_atomic32_inc(&cache_resource->refcnt);
3505         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
3506         dev_flow->dv.modify_hdr = cache_resource;
3507         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
3508                 (void *)cache_resource,
3509                 rte_atomic32_read(&cache_resource->refcnt));
3510         return 0;
3511 }
3512
3513 #define MLX5_CNT_CONTAINER_RESIZE 64
3514
3515 /**
3516  * Get or create a flow counter.
3517  *
3518  * @param[in] dev
3519  *   Pointer to the Ethernet device structure.
3520  * @param[in] shared
3521  *   Indicate if this counter is shared with other flows.
3522  * @param[in] id
3523  *   Counter identifier.
3524  *
3525  * @return
3526  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
3527  */
3528 static struct mlx5_flow_counter *
3529 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
3530                                uint32_t id)
3531 {
3532         struct mlx5_priv *priv = dev->data->dev_private;
3533         struct mlx5_flow_counter *cnt = NULL;
3534         struct mlx5_devx_obj *dcs = NULL;
3535
3536         if (!priv->config.devx) {
3537                 rte_errno = ENOTSUP;
3538                 return NULL;
3539         }
3540         if (shared) {
3541                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
3542                         if (cnt->shared && cnt->id == id) {
3543                                 cnt->ref_cnt++;
3544                                 return cnt;
3545                         }
3546                 }
3547         }
3548         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
3549         if (!dcs)
3550                 return NULL;
3551         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
3552         if (!cnt) {
3553                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
3554                 rte_errno = ENOMEM;
3555                 return NULL;
3556         }
3557         struct mlx5_flow_counter tmpl = {
3558                 .shared = shared,
3559                 .ref_cnt = 1,
3560                 .id = id,
3561                 .dcs = dcs,
3562         };
3563         tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
3564         if (!tmpl.action) {
3565                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
3566                 rte_errno = errno;
3567                 rte_free(cnt);
3568                 return NULL;
3569         }
3570         *cnt = tmpl;
3571         TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
3572         return cnt;
3573 }
3574
3575 /**
3576  * Release a flow counter.
3577  *
3578  * @param[in] dev
3579  *   Pointer to the Ethernet device structure.
3580  * @param[in] counter
3581  *   Pointer to the counter handler.
3582  */
3583 static void
3584 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
3585                                  struct mlx5_flow_counter *counter)
3586 {
3587         struct mlx5_priv *priv = dev->data->dev_private;
3588
3589         if (!counter)
3590                 return;
3591         if (--counter->ref_cnt == 0) {
3592                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
3593                 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
3594                 rte_free(counter);
3595         }
3596 }
3597
3598 /**
3599  * Query a devx flow counter.
3600  *
3601  * @param[in] dev
3602  *   Pointer to the Ethernet device structure.
3603  * @param[in] cnt
3604  *   Pointer to the flow counter.
3605  * @param[out] pkts
3606  *   The statistics value of packets.
3607  * @param[out] bytes
3608  *   The statistics value of bytes.
3609  *
3610  * @return
3611  *   0 on success, otherwise a negative errno value and rte_errno is set.
3612  */
3613 static inline int
3614 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
3615                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
3616                      uint64_t *bytes)
3617 {
3618         return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
3619                                                 0, NULL, NULL, 0);
3620 }
3621
3622 /**
3623  * Get a pool by a counter.
3624  *
3625  * @param[in] cnt
3626  *   Pointer to the counter.
3627  *
3628  * @return
3629  *   The counter pool.
3630  */
3631 static struct mlx5_flow_counter_pool *
3632 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
3633 {
3634         if (!cnt->batch) {
3635                 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
3636                 return (struct mlx5_flow_counter_pool *)cnt - 1;
3637         }
3638         return cnt->pool;
3639 }
3640
3641 /**
3642  * Get a pool by devx counter ID.
3643  *
3644  * @param[in] cont
3645  *   Pointer to the counter container.
3646  * @param[in] id
3647  *   The counter devx ID.
3648  *
3649  * @return
3650  *   The counter pool pointer if exists, NULL otherwise,
3651  */
3652 static struct mlx5_flow_counter_pool *
3653 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
3654 {
3655         struct mlx5_flow_counter_pool *pool;
3656
3657         TAILQ_FOREACH(pool, &cont->pool_list, next) {
3658                 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
3659                                 MLX5_COUNTERS_PER_POOL;
3660
3661                 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
3662                         return pool;
3663         };
3664         return NULL;
3665 }
3666
3667 /**
3668  * Allocate a new memory for the counter values wrapped by all the needed
3669  * management.
3670  *
3671  * @param[in] dev
3672  *   Pointer to the Ethernet device structure.
3673  * @param[in] raws_n
3674  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
3675  *
3676  * @return
3677  *   The new memory management pointer on success, otherwise NULL and rte_errno
3678  *   is set.
3679  */
3680 static struct mlx5_counter_stats_mem_mng *
3681 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
3682 {
3683         struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
3684                                         (dev->data->dev_private))->sh;
3685         struct mlx5_devx_mkey_attr mkey_attr;
3686         struct mlx5_counter_stats_mem_mng *mem_mng;
3687         volatile struct flow_counter_stats *raw_data;
3688         int size = (sizeof(struct flow_counter_stats) *
3689                         MLX5_COUNTERS_PER_POOL +
3690                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
3691                         sizeof(struct mlx5_counter_stats_mem_mng);
3692         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
3693         int i;
3694
3695         if (!mem) {
3696                 rte_errno = ENOMEM;
3697                 return NULL;
3698         }
3699         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
3700         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
3701         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
3702                                                  IBV_ACCESS_LOCAL_WRITE);
3703         if (!mem_mng->umem) {
3704                 rte_errno = errno;
3705                 rte_free(mem);
3706                 return NULL;
3707         }
3708         mkey_attr.addr = (uintptr_t)mem;
3709         mkey_attr.size = size;
3710         mkey_attr.umem_id = mem_mng->umem->umem_id;
3711         mkey_attr.pd = sh->pdn;
3712         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
3713         if (!mem_mng->dm) {
3714                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
3715                 rte_errno = errno;
3716                 rte_free(mem);
3717                 return NULL;
3718         }
3719         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
3720         raw_data = (volatile struct flow_counter_stats *)mem;
3721         for (i = 0; i < raws_n; ++i) {
3722                 mem_mng->raws[i].mem_mng = mem_mng;
3723                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
3724         }
3725         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
3726         return mem_mng;
3727 }
3728
3729 /**
3730  * Resize a counter container.
3731  *
3732  * @param[in] dev
3733  *   Pointer to the Ethernet device structure.
3734  * @param[in] batch
3735  *   Whether the pool is for counter that was allocated by batch command.
3736  *
3737  * @return
3738  *   The new container pointer on success, otherwise NULL and rte_errno is set.
3739  */
3740 static struct mlx5_pools_container *
3741 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
3742 {
3743         struct mlx5_priv *priv = dev->data->dev_private;
3744         struct mlx5_pools_container *cont =
3745                         MLX5_CNT_CONTAINER(priv->sh, batch, 0);
3746         struct mlx5_pools_container *new_cont =
3747                         MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
3748         struct mlx5_counter_stats_mem_mng *mem_mng;
3749         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
3750         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
3751         int i;
3752
3753         if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
3754                 /* The last resize still hasn't detected by the host thread. */
3755                 rte_errno = EAGAIN;
3756                 return NULL;
3757         }
3758         new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
3759         if (!new_cont->pools) {
3760                 rte_errno = ENOMEM;
3761                 return NULL;
3762         }
3763         if (cont->n)
3764                 memcpy(new_cont->pools, cont->pools, cont->n *
3765                        sizeof(struct mlx5_flow_counter_pool *));
3766         mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
3767                 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
3768         if (!mem_mng) {
3769                 rte_free(new_cont->pools);
3770                 return NULL;
3771         }
3772         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
3773                 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
3774                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
3775                                  i, next);
3776         new_cont->n = resize;
3777         rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
3778         TAILQ_INIT(&new_cont->pool_list);
3779         TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
3780         new_cont->init_mem_mng = mem_mng;
3781         rte_cio_wmb();
3782          /* Flip the master container. */
3783         priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
3784         return new_cont;
3785 }
3786
3787 /**
3788  * Query a devx flow counter.
3789  *
3790  * @param[in] dev
3791  *   Pointer to the Ethernet device structure.
3792  * @param[in] cnt
3793  *   Pointer to the flow counter.
3794  * @param[out] pkts
3795  *   The statistics value of packets.
3796  * @param[out] bytes
3797  *   The statistics value of bytes.
3798  *
3799  * @return
3800  *   0 on success, otherwise a negative errno value and rte_errno is set.
3801  */
3802 static inline int
3803 _flow_dv_query_count(struct rte_eth_dev *dev,
3804                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
3805                      uint64_t *bytes)
3806 {
3807         struct mlx5_priv *priv = dev->data->dev_private;
3808         struct mlx5_flow_counter_pool *pool =
3809                         flow_dv_counter_pool_get(cnt);
3810         int offset = cnt - &pool->counters_raw[0];
3811
3812         if (priv->counter_fallback)
3813                 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
3814
3815         rte_spinlock_lock(&pool->sl);
3816         /*
3817          * The single counters allocation may allocate smaller ID than the
3818          * current allocated in parallel to the host reading.
3819          * In this case the new counter values must be reported as 0.
3820          */
3821         if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
3822                 *pkts = 0;
3823                 *bytes = 0;
3824         } else {
3825                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
3826                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
3827         }
3828         rte_spinlock_unlock(&pool->sl);
3829         return 0;
3830 }
3831
3832 /**
3833  * Create and initialize a new counter pool.
3834  *
3835  * @param[in] dev
3836  *   Pointer to the Ethernet device structure.
3837  * @param[out] dcs
3838  *   The devX counter handle.
3839  * @param[in] batch
3840  *   Whether the pool is for counter that was allocated by batch command.
3841  *
3842  * @return
3843  *   A new pool pointer on success, NULL otherwise and rte_errno is set.
3844  */
3845 static struct mlx5_flow_counter_pool *
3846 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
3847                     uint32_t batch)
3848 {
3849         struct mlx5_priv *priv = dev->data->dev_private;
3850         struct mlx5_flow_counter_pool *pool;
3851         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
3852                                                                0);
3853         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
3854         uint32_t size;
3855
3856         if (cont->n == n_valid) {
3857                 cont = flow_dv_container_resize(dev, batch);
3858                 if (!cont)
3859                         return NULL;
3860         }
3861         size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
3862                         sizeof(struct mlx5_flow_counter);
3863         pool = rte_calloc(__func__, 1, size, 0);
3864         if (!pool) {
3865                 rte_errno = ENOMEM;
3866                 return NULL;
3867         }
3868         pool->min_dcs = dcs;
3869         pool->raw = cont->init_mem_mng->raws + n_valid %
3870                                                      MLX5_CNT_CONTAINER_RESIZE;
3871         pool->raw_hw = NULL;
3872         rte_spinlock_init(&pool->sl);
3873         /*
3874          * The generation of the new allocated counters in this pool is 0, 2 in
3875          * the pool generation makes all the counters valid for allocation.
3876          */
3877         rte_atomic64_set(&pool->query_gen, 0x2);
3878         TAILQ_INIT(&pool->counters);
3879         TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
3880         cont->pools[n_valid] = pool;
3881         /* Pool initialization must be updated before host thread access. */
3882         rte_cio_wmb();
3883         rte_atomic16_add(&cont->n_valid, 1);
3884         return pool;
3885 }
3886
3887 /**
3888  * Prepare a new counter and/or a new counter pool.
3889  *
3890  * @param[in] dev
3891  *   Pointer to the Ethernet device structure.
3892  * @param[out] cnt_free
3893  *   Where to put the pointer of a new counter.
3894  * @param[in] batch
3895  *   Whether the pool is for counter that was allocated by batch command.
3896  *
3897  * @return
3898  *   The free counter pool pointer and @p cnt_free is set on success,
3899  *   NULL otherwise and rte_errno is set.
3900  */
3901 static struct mlx5_flow_counter_pool *
3902 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
3903                              struct mlx5_flow_counter **cnt_free,
3904                              uint32_t batch)
3905 {
3906         struct mlx5_priv *priv = dev->data->dev_private;
3907         struct mlx5_flow_counter_pool *pool;
3908         struct mlx5_devx_obj *dcs = NULL;
3909         struct mlx5_flow_counter *cnt;
3910         uint32_t i;
3911
3912         if (!batch) {
3913                 /* bulk_bitmap must be 0 for single counter allocation. */
3914                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
3915                 if (!dcs)
3916                         return NULL;
3917                 pool = flow_dv_find_pool_by_id
3918                         (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
3919                 if (!pool) {
3920                         pool = flow_dv_pool_create(dev, dcs, batch);
3921                         if (!pool) {
3922                                 mlx5_devx_cmd_destroy(dcs);
3923                                 return NULL;
3924                         }
3925                 } else if (dcs->id < pool->min_dcs->id) {
3926                         rte_atomic64_set(&pool->a64_dcs,
3927                                          (int64_t)(uintptr_t)dcs);
3928                 }
3929                 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
3930                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
3931                 cnt->dcs = dcs;
3932                 *cnt_free = cnt;
3933                 return pool;
3934         }
3935         /* bulk_bitmap is in 128 counters units. */
3936         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
3937                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
3938         if (!dcs) {
3939                 rte_errno = ENODATA;
3940                 return NULL;
3941         }
3942         pool = flow_dv_pool_create(dev, dcs, batch);
3943         if (!pool) {
3944                 mlx5_devx_cmd_destroy(dcs);
3945                 return NULL;
3946         }
3947         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
3948                 cnt = &pool->counters_raw[i];
3949                 cnt->pool = pool;
3950                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
3951         }
3952         *cnt_free = &pool->counters_raw[0];
3953         return pool;
3954 }
3955
3956 /**
3957  * Search for existed shared counter.
3958  *
3959  * @param[in] cont
3960  *   Pointer to the relevant counter pool container.
3961  * @param[in] id
3962  *   The shared counter ID to search.
3963  *
3964  * @return
3965  *   NULL if not existed, otherwise pointer to the shared counter.
3966  */
3967 static struct mlx5_flow_counter *
3968 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
3969                               uint32_t id)
3970 {
3971         static struct mlx5_flow_counter *cnt;
3972         struct mlx5_flow_counter_pool *pool;
3973         int i;
3974
3975         TAILQ_FOREACH(pool, &cont->pool_list, next) {
3976                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
3977                         cnt = &pool->counters_raw[i];
3978                         if (cnt->ref_cnt && cnt->shared && cnt->id == id)
3979                                 return cnt;
3980                 }
3981         }
3982         return NULL;
3983 }
3984
3985 /**
3986  * Allocate a flow counter.
3987  *
3988  * @param[in] dev
3989  *   Pointer to the Ethernet device structure.
3990  * @param[in] shared
3991  *   Indicate if this counter is shared with other flows.
3992  * @param[in] id
3993  *   Counter identifier.
3994  * @param[in] group
3995  *   Counter flow group.
3996  *
3997  * @return
3998  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
3999  */
4000 static struct mlx5_flow_counter *
4001 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4002                       uint16_t group)
4003 {
4004         struct mlx5_priv *priv = dev->data->dev_private;
4005         struct mlx5_flow_counter_pool *pool = NULL;
4006         struct mlx5_flow_counter *cnt_free = NULL;
4007         /*
4008          * Currently group 0 flow counter cannot be assigned to a flow if it is
4009          * not the first one in the batch counter allocation, so it is better
4010          * to allocate counters one by one for these flows in a separate
4011          * container.
4012          * A counter can be shared between different groups so need to take
4013          * shared counters from the single container.
4014          */
4015         uint32_t batch = (group && !shared) ? 1 : 0;
4016         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4017                                                                0);
4018
4019         if (priv->counter_fallback)
4020                 return flow_dv_counter_alloc_fallback(dev, shared, id);
4021         if (!priv->config.devx) {
4022                 rte_errno = ENOTSUP;
4023                 return NULL;
4024         }
4025         if (shared) {
4026                 cnt_free = flow_dv_counter_shared_search(cont, id);
4027                 if (cnt_free) {
4028                         if (cnt_free->ref_cnt + 1 == 0) {
4029                                 rte_errno = E2BIG;
4030                                 return NULL;
4031                         }
4032                         cnt_free->ref_cnt++;
4033                         return cnt_free;
4034                 }
4035         }
4036         /* Pools which has a free counters are in the start. */
4037         TAILQ_FOREACH(pool, &cont->pool_list, next) {
4038                 /*
4039                  * The free counter reset values must be updated between the
4040                  * counter release to the counter allocation, so, at least one
4041                  * query must be done in this time. ensure it by saving the
4042                  * query generation in the release time.
4043                  * The free list is sorted according to the generation - so if
4044                  * the first one is not updated, all the others are not
4045                  * updated too.
4046                  */
4047                 cnt_free = TAILQ_FIRST(&pool->counters);
4048                 if (cnt_free && cnt_free->query_gen + 1 <
4049                     rte_atomic64_read(&pool->query_gen))
4050                         break;
4051                 cnt_free = NULL;
4052         }
4053         if (!cnt_free) {
4054                 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
4055                 if (!pool)
4056                         return NULL;
4057         }
4058         cnt_free->batch = batch;
4059         /* Create a DV counter action only in the first time usage. */
4060         if (!cnt_free->action) {
4061                 uint16_t offset;
4062                 struct mlx5_devx_obj *dcs;
4063
4064                 if (batch) {
4065                         offset = cnt_free - &pool->counters_raw[0];
4066                         dcs = pool->min_dcs;
4067                 } else {
4068                         offset = 0;
4069                         dcs = cnt_free->dcs;
4070                 }
4071                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
4072                                         (dcs->obj, offset);
4073                 if (!cnt_free->action) {
4074                         rte_errno = errno;
4075                         return NULL;
4076                 }
4077         }
4078         /* Update the counter reset values. */
4079         if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
4080                                  &cnt_free->bytes))
4081                 return NULL;
4082         cnt_free->shared = shared;
4083         cnt_free->ref_cnt = 1;
4084         cnt_free->id = id;
4085         if (!priv->sh->cmng.query_thread_on)
4086                 /* Start the asynchronous batch query by the host thread. */
4087                 mlx5_set_query_alarm(priv->sh);
4088         TAILQ_REMOVE(&pool->counters, cnt_free, next);
4089         if (TAILQ_EMPTY(&pool->counters)) {
4090                 /* Move the pool to the end of the container pool list. */
4091                 TAILQ_REMOVE(&cont->pool_list, pool, next);
4092                 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
4093         }
4094         return cnt_free;
4095 }
4096
4097 /**
4098  * Release a flow counter.
4099  *
4100  * @param[in] dev
4101  *   Pointer to the Ethernet device structure.
4102  * @param[in] counter
4103  *   Pointer to the counter handler.
4104  */
4105 static void
4106 flow_dv_counter_release(struct rte_eth_dev *dev,
4107                         struct mlx5_flow_counter *counter)
4108 {
4109         struct mlx5_priv *priv = dev->data->dev_private;
4110
4111         if (!counter)
4112                 return;
4113         if (priv->counter_fallback) {
4114                 flow_dv_counter_release_fallback(dev, counter);
4115                 return;
4116         }
4117         if (--counter->ref_cnt == 0) {
4118                 struct mlx5_flow_counter_pool *pool =
4119                                 flow_dv_counter_pool_get(counter);
4120
4121                 /* Put the counter in the end - the last updated one. */
4122                 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
4123                 counter->query_gen = rte_atomic64_read(&pool->query_gen);
4124         }
4125 }
4126
4127 /**
4128  * Verify the @p attributes will be correctly understood by the NIC and store
4129  * them in the @p flow if everything is correct.
4130  *
4131  * @param[in] dev
4132  *   Pointer to dev struct.
4133  * @param[in] attributes
4134  *   Pointer to flow attributes
4135  * @param[in] external
4136  *   This flow rule is created by request external to PMD.
4137  * @param[out] error
4138  *   Pointer to error structure.
4139  *
4140  * @return
4141  *   0 on success, a negative errno value otherwise and rte_errno is set.
4142  */
4143 static int
4144 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4145                             const struct rte_flow_attr *attributes,
4146                             bool external __rte_unused,
4147                             struct rte_flow_error *error)
4148 {
4149         struct mlx5_priv *priv = dev->data->dev_private;
4150         uint32_t priority_max = priv->config.flow_prio - 1;
4151
4152 #ifndef HAVE_MLX5DV_DR
4153         if (attributes->group)
4154                 return rte_flow_error_set(error, ENOTSUP,
4155                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4156                                           NULL,
4157                                           "groups are not supported");
4158 #else
4159         uint32_t table;
4160         int ret;
4161
4162         ret = mlx5_flow_group_to_table(attributes, external,
4163                                        attributes->group,
4164                                        &table, error);
4165         if (ret)
4166                 return ret;
4167 #endif
4168         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4169             attributes->priority >= priority_max)
4170                 return rte_flow_error_set(error, ENOTSUP,
4171                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4172                                           NULL,
4173                                           "priority out of range");
4174         if (attributes->transfer) {
4175                 if (!priv->config.dv_esw_en)
4176                         return rte_flow_error_set
4177                                 (error, ENOTSUP,
4178                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4179                                  "E-Switch dr is not supported");
4180                 if (!(priv->representor || priv->master))
4181                         return rte_flow_error_set
4182                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4183                                  NULL, "E-Switch configuration can only be"
4184                                  " done by a master or a representor device");
4185                 if (attributes->egress)
4186                         return rte_flow_error_set
4187                                 (error, ENOTSUP,
4188                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4189                                  "egress is not supported");
4190         }
4191         if (!(attributes->egress ^ attributes->ingress))
4192                 return rte_flow_error_set(error, ENOTSUP,
4193                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4194                                           "must specify exactly one of "
4195                                           "ingress or egress");
4196         return 0;
4197 }
4198
4199 /**
4200  * Internal validation function. For validating both actions and items.
4201  *
4202  * @param[in] dev
4203  *   Pointer to the rte_eth_dev structure.
4204  * @param[in] attr
4205  *   Pointer to the flow attributes.
4206  * @param[in] items
4207  *   Pointer to the list of items.
4208  * @param[in] actions
4209  *   Pointer to the list of actions.
4210  * @param[in] external
4211  *   This flow rule is created by request external to PMD.
4212  * @param[out] error
4213  *   Pointer to the error structure.
4214  *
4215  * @return
4216  *   0 on success, a negative errno value otherwise and rte_errno is set.
4217  */
4218 static int
4219 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4220                  const struct rte_flow_item items[],
4221                  const struct rte_flow_action actions[],
4222                  bool external, struct rte_flow_error *error)
4223 {
4224         int ret;
4225         uint64_t action_flags = 0;
4226         uint64_t item_flags = 0;
4227         uint64_t last_item = 0;
4228         uint8_t next_protocol = 0xff;
4229         uint16_t ether_type = 0;
4230         int actions_n = 0;
4231         const struct rte_flow_item *gre_item = NULL;
4232         struct rte_flow_item_tcp nic_tcp_mask = {
4233                 .hdr = {
4234                         .tcp_flags = 0xFF,
4235                         .src_port = RTE_BE16(UINT16_MAX),
4236                         .dst_port = RTE_BE16(UINT16_MAX),
4237                 }
4238         };
4239         struct mlx5_priv *priv = dev->data->dev_private;
4240         struct mlx5_dev_config *dev_conf = &priv->config;
4241
4242         if (items == NULL)
4243                 return -1;
4244         ret = flow_dv_validate_attributes(dev, attr, external, error);
4245         if (ret < 0)
4246                 return ret;
4247         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4248                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4249                 int type = items->type;
4250
4251                 switch (type) {
4252                 case RTE_FLOW_ITEM_TYPE_VOID:
4253                         break;
4254                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4255                         ret = flow_dv_validate_item_port_id
4256                                         (dev, items, attr, item_flags, error);
4257                         if (ret < 0)
4258                                 return ret;
4259                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4260                         break;
4261                 case RTE_FLOW_ITEM_TYPE_ETH:
4262                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4263                                                           error);
4264                         if (ret < 0)
4265                                 return ret;
4266                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4267                                              MLX5_FLOW_LAYER_OUTER_L2;
4268                         if (items->mask != NULL && items->spec != NULL) {
4269                                 ether_type =
4270                                         ((const struct rte_flow_item_eth *)
4271                                          items->spec)->type;
4272                                 ether_type &=
4273                                         ((const struct rte_flow_item_eth *)
4274                                          items->mask)->type;
4275                                 ether_type = rte_be_to_cpu_16(ether_type);
4276                         } else {
4277                                 ether_type = 0;
4278                         }
4279                         break;
4280                 case RTE_FLOW_ITEM_TYPE_VLAN:
4281                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
4282                                                            dev, error);
4283                         if (ret < 0)
4284                                 return ret;
4285                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
4286                                              MLX5_FLOW_LAYER_OUTER_VLAN;
4287                         if (items->mask != NULL && items->spec != NULL) {
4288                                 ether_type =
4289                                         ((const struct rte_flow_item_vlan *)
4290                                          items->spec)->inner_type;
4291                                 ether_type &=
4292                                         ((const struct rte_flow_item_vlan *)
4293                                          items->mask)->inner_type;
4294                                 ether_type = rte_be_to_cpu_16(ether_type);
4295                         } else {
4296                                 ether_type = 0;
4297                         }
4298                         break;
4299                 case RTE_FLOW_ITEM_TYPE_IPV4:
4300                         mlx5_flow_tunnel_ip_check(items, next_protocol,
4301                                                   &item_flags, &tunnel);
4302                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
4303                                                            last_item,
4304                                                            ether_type, NULL,
4305                                                            error);
4306                         if (ret < 0)
4307                                 return ret;
4308                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4309                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4310                         if (items->mask != NULL &&
4311                             ((const struct rte_flow_item_ipv4 *)
4312                              items->mask)->hdr.next_proto_id) {
4313                                 next_protocol =
4314                                         ((const struct rte_flow_item_ipv4 *)
4315                                          (items->spec))->hdr.next_proto_id;
4316                                 next_protocol &=
4317                                         ((const struct rte_flow_item_ipv4 *)
4318                                          (items->mask))->hdr.next_proto_id;
4319                         } else {
4320                                 /* Reset for inner layer. */
4321                                 next_protocol = 0xff;
4322                         }
4323                         break;
4324                 case RTE_FLOW_ITEM_TYPE_IPV6:
4325                         mlx5_flow_tunnel_ip_check(items, next_protocol,
4326                                                   &item_flags, &tunnel);
4327                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
4328                                                            last_item,
4329                                                            ether_type, NULL,
4330                                                            error);
4331                         if (ret < 0)
4332                                 return ret;
4333                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4334                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4335                         if (items->mask != NULL &&
4336                             ((const struct rte_flow_item_ipv6 *)
4337                              items->mask)->hdr.proto) {
4338                                 next_protocol =
4339                                         ((const struct rte_flow_item_ipv6 *)
4340                                          items->spec)->hdr.proto;
4341                                 next_protocol &=
4342                                         ((const struct rte_flow_item_ipv6 *)
4343                                          items->mask)->hdr.proto;
4344                         } else {
4345                                 /* Reset for inner layer. */
4346                                 next_protocol = 0xff;
4347                         }
4348                         break;
4349                 case RTE_FLOW_ITEM_TYPE_TCP:
4350                         ret = mlx5_flow_validate_item_tcp
4351                                                 (items, item_flags,
4352                                                  next_protocol,
4353                                                  &nic_tcp_mask,
4354                                                  error);
4355                         if (ret < 0)
4356                                 return ret;
4357                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
4358                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
4359                         break;
4360                 case RTE_FLOW_ITEM_TYPE_UDP:
4361                         ret = mlx5_flow_validate_item_udp(items, item_flags,
4362                                                           next_protocol,
4363                                                           error);
4364                         if (ret < 0)
4365                                 return ret;
4366                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
4367                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
4368                         break;
4369                 case RTE_FLOW_ITEM_TYPE_GRE:
4370                         ret = mlx5_flow_validate_item_gre(items, item_flags,
4371                                                           next_protocol, error);
4372                         if (ret < 0)
4373                                 return ret;
4374                         gre_item = items;
4375                         last_item = MLX5_FLOW_LAYER_GRE;
4376                         break;
4377                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4378                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
4379                                                             next_protocol,
4380                                                             error);
4381                         if (ret < 0)
4382                                 return ret;
4383                         last_item = MLX5_FLOW_LAYER_NVGRE;
4384                         break;
4385                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
4386                         ret = mlx5_flow_validate_item_gre_key
4387                                 (items, item_flags, gre_item, error);
4388                         if (ret < 0)
4389                                 return ret;
4390                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
4391                         break;
4392                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4393                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
4394                                                             error);
4395                         if (ret < 0)
4396                                 return ret;
4397                         last_item = MLX5_FLOW_LAYER_VXLAN;
4398                         break;
4399                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4400                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
4401                                                                 item_flags, dev,
4402                                                                 error);
4403                         if (ret < 0)
4404                                 return ret;
4405                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
4406                         break;
4407                 case RTE_FLOW_ITEM_TYPE_GENEVE:
4408                         ret = mlx5_flow_validate_item_geneve(items,
4409                                                              item_flags, dev,
4410                                                              error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
4414                         break;
4415                 case RTE_FLOW_ITEM_TYPE_MPLS:
4416                         ret = mlx5_flow_validate_item_mpls(dev, items,
4417                                                            item_flags,
4418                                                            last_item, error);
4419                         if (ret < 0)
4420                                 return ret;
4421                         last_item = MLX5_FLOW_LAYER_MPLS;
4422                         break;
4423
4424                 case RTE_FLOW_ITEM_TYPE_MARK:
4425                         ret = flow_dv_validate_item_mark(dev, items, attr,
4426                                                          error);
4427                         if (ret < 0)
4428                                 return ret;
4429                         last_item = MLX5_FLOW_ITEM_MARK;
4430                         break;
4431                 case RTE_FLOW_ITEM_TYPE_META:
4432                         ret = flow_dv_validate_item_meta(dev, items, attr,
4433                                                          error);
4434                         if (ret < 0)
4435                                 return ret;
4436                         last_item = MLX5_FLOW_ITEM_METADATA;
4437                         break;
4438                 case RTE_FLOW_ITEM_TYPE_ICMP:
4439                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
4440                                                            next_protocol,
4441                                                            error);
4442                         if (ret < 0)
4443                                 return ret;
4444                         last_item = MLX5_FLOW_LAYER_ICMP;
4445                         break;
4446                 case RTE_FLOW_ITEM_TYPE_ICMP6:
4447                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
4448                                                             next_protocol,
4449                                                             error);
4450                         if (ret < 0)
4451                                 return ret;
4452                         last_item = MLX5_FLOW_LAYER_ICMP6;
4453                         break;
4454                 case RTE_FLOW_ITEM_TYPE_TAG:
4455                         ret = flow_dv_validate_item_tag(dev, items,
4456                                                         attr, error);
4457                         if (ret < 0)
4458                                 return ret;
4459                         last_item = MLX5_FLOW_ITEM_TAG;
4460                         break;
4461                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
4462                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
4463                         break;
4464                 default:
4465                         return rte_flow_error_set(error, ENOTSUP,
4466                                                   RTE_FLOW_ERROR_TYPE_ITEM,
4467                                                   NULL, "item not supported");
4468                 }
4469                 item_flags |= last_item;
4470         }
4471         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4472                 int type = actions->type;
4473                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4474                         return rte_flow_error_set(error, ENOTSUP,
4475                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4476                                                   actions, "too many actions");
4477                 switch (type) {
4478                 case RTE_FLOW_ACTION_TYPE_VOID:
4479                         break;
4480                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4481                         ret = flow_dv_validate_action_port_id(dev,
4482                                                               action_flags,
4483                                                               actions,
4484                                                               attr,
4485                                                               error);
4486                         if (ret)
4487                                 return ret;
4488                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4489                         ++actions_n;
4490                         break;
4491                 case RTE_FLOW_ACTION_TYPE_FLAG:
4492                         ret = flow_dv_validate_action_flag(dev, action_flags,
4493                                                            attr, error);
4494                         if (ret < 0)
4495                                 return ret;
4496                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
4497                                 /* Count all modify-header actions as one. */
4498                                 if (!(action_flags &
4499                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
4500                                         ++actions_n;
4501                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
4502                                                 MLX5_FLOW_ACTION_MARK_EXT;
4503                         } else {
4504                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
4505                                 ++actions_n;
4506                         }
4507                         break;
4508                 case RTE_FLOW_ACTION_TYPE_MARK:
4509                         ret = flow_dv_validate_action_mark(dev, actions,
4510                                                            action_flags,
4511                                                            attr, error);
4512                         if (ret < 0)
4513                                 return ret;
4514                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
4515                                 /* Count all modify-header actions as one. */
4516                                 if (!(action_flags &
4517                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
4518                                         ++actions_n;
4519                                 action_flags |= MLX5_FLOW_ACTION_MARK |
4520                                                 MLX5_FLOW_ACTION_MARK_EXT;
4521                         } else {
4522                                 action_flags |= MLX5_FLOW_ACTION_MARK;
4523                                 ++actions_n;
4524                         }
4525                         break;
4526                 case RTE_FLOW_ACTION_TYPE_SET_META:
4527                         ret = flow_dv_validate_action_set_meta(dev, actions,
4528                                                                action_flags,
4529                                                                attr, error);
4530                         if (ret < 0)
4531                                 return ret;
4532                         /* Count all modify-header actions as one action. */
4533                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4534                                 ++actions_n;
4535                         action_flags |= MLX5_FLOW_ACTION_SET_META;
4536                         break;
4537                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
4538                         ret = flow_dv_validate_action_set_tag(dev, actions,
4539                                                               action_flags,
4540                                                               attr, error);
4541                         if (ret < 0)
4542                                 return ret;
4543                         /* Count all modify-header actions as one action. */
4544                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4545                                 ++actions_n;
4546                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
4547                         break;
4548                 case RTE_FLOW_ACTION_TYPE_DROP:
4549                         ret = mlx5_flow_validate_action_drop(action_flags,
4550                                                              attr, error);
4551                         if (ret < 0)
4552                                 return ret;
4553                         action_flags |= MLX5_FLOW_ACTION_DROP;
4554                         ++actions_n;
4555                         break;
4556                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4557                         ret = mlx5_flow_validate_action_queue(actions,
4558                                                               action_flags, dev,
4559                                                               attr, error);
4560                         if (ret < 0)
4561                                 return ret;
4562                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
4563                         ++actions_n;
4564                         break;
4565                 case RTE_FLOW_ACTION_TYPE_RSS:
4566                         ret = mlx5_flow_validate_action_rss(actions,
4567                                                             action_flags, dev,
4568                                                             attr, item_flags,
4569                                                             error);
4570                         if (ret < 0)
4571                                 return ret;
4572                         action_flags |= MLX5_FLOW_ACTION_RSS;
4573                         ++actions_n;
4574                         break;
4575                 case RTE_FLOW_ACTION_TYPE_COUNT:
4576                         ret = flow_dv_validate_action_count(dev, error);
4577                         if (ret < 0)
4578                                 return ret;
4579                         action_flags |= MLX5_FLOW_ACTION_COUNT;
4580                         ++actions_n;
4581                         break;
4582                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
4583                         if (flow_dv_validate_action_pop_vlan(dev,
4584                                                              action_flags,
4585                                                              actions,
4586                                                              item_flags, attr,
4587                                                              error))
4588                                 return -rte_errno;
4589                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
4590                         ++actions_n;
4591                         break;
4592                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4593                         ret = flow_dv_validate_action_push_vlan(action_flags,
4594                                                                 item_flags,
4595                                                                 actions, attr,
4596                                                                 error);
4597                         if (ret < 0)
4598                                 return ret;
4599                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
4600                         ++actions_n;
4601                         break;
4602                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4603                         ret = flow_dv_validate_action_set_vlan_pcp
4604                                                 (action_flags, actions, error);
4605                         if (ret < 0)
4606                                 return ret;
4607                         /* Count PCP with push_vlan command. */
4608                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
4609                         break;
4610                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4611                         ret = flow_dv_validate_action_set_vlan_vid
4612                                                 (item_flags, action_flags,
4613                                                  actions, error);
4614                         if (ret < 0)
4615                                 return ret;
4616                         /* Count VID with push_vlan command. */
4617                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
4618                         break;
4619                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4620                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4621                         ret = flow_dv_validate_action_l2_encap(action_flags,
4622                                                                actions, attr,
4623                                                                error);
4624                         if (ret < 0)
4625                                 return ret;
4626                         action_flags |= actions->type ==
4627                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
4628                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
4629                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
4630                         ++actions_n;
4631                         break;
4632                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4633                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4634                         ret = flow_dv_validate_action_l2_decap(action_flags,
4635                                                                attr, error);
4636                         if (ret < 0)
4637                                 return ret;
4638                         action_flags |= actions->type ==
4639                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
4640                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
4641                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
4642                         ++actions_n;
4643                         break;
4644                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4645                         ret = flow_dv_validate_action_raw_encap(action_flags,
4646                                                                 actions, attr,
4647                                                                 error);
4648                         if (ret < 0)
4649                                 return ret;
4650                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
4651                         ++actions_n;
4652                         break;
4653                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4654                         ret = flow_dv_validate_action_raw_decap(action_flags,
4655                                                                 actions, attr,
4656                                                                 error);
4657                         if (ret < 0)
4658                                 return ret;
4659                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
4660                         ++actions_n;
4661                         break;
4662                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4663                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4664                         ret = flow_dv_validate_action_modify_mac(action_flags,
4665                                                                  actions,
4666                                                                  item_flags,
4667                                                                  error);
4668                         if (ret < 0)
4669                                 return ret;
4670                         /* Count all modify-header actions as one action. */
4671                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4672                                 ++actions_n;
4673                         action_flags |= actions->type ==
4674                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
4675                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
4676                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
4677                         break;
4678
4679                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4680                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4681                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
4682                                                                   actions,
4683                                                                   item_flags,
4684                                                                   error);
4685                         if (ret < 0)
4686                                 return ret;
4687                         /* Count all modify-header actions as one action. */
4688                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4689                                 ++actions_n;
4690                         action_flags |= actions->type ==
4691                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
4692                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
4693                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
4694                         break;
4695                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4696                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4697                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
4698                                                                   actions,
4699                                                                   item_flags,
4700                                                                   error);
4701                         if (ret < 0)
4702                                 return ret;
4703                         /* Count all modify-header actions as one action. */
4704                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4705                                 ++actions_n;
4706                         action_flags |= actions->type ==
4707                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
4708                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
4709                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
4710                         break;
4711                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4712                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4713                         ret = flow_dv_validate_action_modify_tp(action_flags,
4714                                                                 actions,
4715                                                                 item_flags,
4716                                                                 error);
4717                         if (ret < 0)
4718                                 return ret;
4719                         /* Count all modify-header actions as one action. */
4720                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4721                                 ++actions_n;
4722                         action_flags |= actions->type ==
4723                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
4724                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
4725                                                 MLX5_FLOW_ACTION_SET_TP_DST;
4726                         break;
4727                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4728                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
4729                         ret = flow_dv_validate_action_modify_ttl(action_flags,
4730                                                                  actions,
4731                                                                  item_flags,
4732                                                                  error);
4733                         if (ret < 0)
4734                                 return ret;
4735                         /* Count all modify-header actions as one action. */
4736                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4737                                 ++actions_n;
4738                         action_flags |= actions->type ==
4739                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
4740                                                 MLX5_FLOW_ACTION_SET_TTL :
4741                                                 MLX5_FLOW_ACTION_DEC_TTL;
4742                         break;
4743                 case RTE_FLOW_ACTION_TYPE_JUMP:
4744                         ret = flow_dv_validate_action_jump(actions,
4745                                                            action_flags,
4746                                                            attr, external,
4747                                                            error);
4748                         if (ret)
4749                                 return ret;
4750                         ++actions_n;
4751                         action_flags |= MLX5_FLOW_ACTION_JUMP;
4752                         break;
4753                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4754                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4755                         ret = flow_dv_validate_action_modify_tcp_seq
4756                                                                 (action_flags,
4757                                                                  actions,
4758                                                                  item_flags,
4759                                                                  error);
4760                         if (ret < 0)
4761                                 return ret;
4762                         /* Count all modify-header actions as one action. */
4763                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4764                                 ++actions_n;
4765                         action_flags |= actions->type ==
4766                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
4767                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
4768                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
4769                         break;
4770                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4771                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4772                         ret = flow_dv_validate_action_modify_tcp_ack
4773                                                                 (action_flags,
4774                                                                  actions,
4775                                                                  item_flags,
4776                                                                  error);
4777                         if (ret < 0)
4778                                 return ret;
4779                         /* Count all modify-header actions as one action. */
4780                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4781                                 ++actions_n;
4782                         action_flags |= actions->type ==
4783                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
4784                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
4785                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
4786                         break;
4787                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
4788                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
4789                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
4790                         break;
4791                 case RTE_FLOW_ACTION_TYPE_METER:
4792                         ret = mlx5_flow_validate_action_meter(dev,
4793                                                               action_flags,
4794                                                               actions, attr,
4795                                                               error);
4796                         if (ret < 0)
4797                                 return ret;
4798                         action_flags |= MLX5_FLOW_ACTION_METER;
4799                         ++actions_n;
4800                         break;
4801                 default:
4802                         return rte_flow_error_set(error, ENOTSUP,
4803                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4804                                                   actions,
4805                                                   "action not supported");
4806                 }
4807         }
4808         if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) &&
4809             (action_flags & MLX5_FLOW_VLAN_ACTIONS))
4810                 return rte_flow_error_set(error, ENOTSUP,
4811                                           RTE_FLOW_ERROR_TYPE_ACTION,
4812                                           actions,
4813                                           "can't have vxlan and vlan"
4814                                           " actions in the same rule");
4815         /* Eswitch has few restrictions on using items and actions */
4816         if (attr->transfer) {
4817                 if (!mlx5_flow_ext_mreg_supported(dev) &&
4818                     action_flags & MLX5_FLOW_ACTION_FLAG)
4819                         return rte_flow_error_set(error, ENOTSUP,
4820                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4821                                                   NULL,
4822                                                   "unsupported action FLAG");
4823                 if (!mlx5_flow_ext_mreg_supported(dev) &&
4824                     action_flags & MLX5_FLOW_ACTION_MARK)
4825                         return rte_flow_error_set(error, ENOTSUP,
4826                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4827                                                   NULL,
4828                                                   "unsupported action MARK");
4829                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
4830                         return rte_flow_error_set(error, ENOTSUP,
4831                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4832                                                   NULL,
4833                                                   "unsupported action QUEUE");
4834                 if (action_flags & MLX5_FLOW_ACTION_RSS)
4835                         return rte_flow_error_set(error, ENOTSUP,
4836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4837                                                   NULL,
4838                                                   "unsupported action RSS");
4839                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4840                         return rte_flow_error_set(error, EINVAL,
4841                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4842                                                   actions,
4843                                                   "no fate action is found");
4844         } else {
4845                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
4846                         return rte_flow_error_set(error, EINVAL,
4847                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4848                                                   actions,
4849                                                   "no fate action is found");
4850         }
4851         return 0;
4852 }
4853
4854 /**
4855  * Internal preparation function. Allocates the DV flow size,
4856  * this size is constant.
4857  *
4858  * @param[in] attr
4859  *   Pointer to the flow attributes.
4860  * @param[in] items
4861  *   Pointer to the list of items.
4862  * @param[in] actions
4863  *   Pointer to the list of actions.
4864  * @param[out] error
4865  *   Pointer to the error structure.
4866  *
4867  * @return
4868  *   Pointer to mlx5_flow object on success,
4869  *   otherwise NULL and rte_errno is set.
4870  */
4871 static struct mlx5_flow *
4872 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
4873                 const struct rte_flow_item items[] __rte_unused,
4874                 const struct rte_flow_action actions[] __rte_unused,
4875                 struct rte_flow_error *error)
4876 {
4877         size_t size = sizeof(struct mlx5_flow);
4878         struct mlx5_flow *dev_flow;
4879
4880         dev_flow = rte_calloc(__func__, 1, size, 0);
4881         if (!dev_flow) {
4882                 rte_flow_error_set(error, ENOMEM,
4883                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4884                                    "not enough memory to create flow");
4885                 return NULL;
4886         }
4887         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
4888         dev_flow->ingress = attr->ingress;
4889         dev_flow->transfer = attr->transfer;
4890         return dev_flow;
4891 }
4892
4893 #ifndef NDEBUG
4894 /**
4895  * Sanity check for match mask and value. Similar to check_valid_spec() in
4896  * kernel driver. If unmasked bit is present in value, it returns failure.
4897  *
4898  * @param match_mask
4899  *   pointer to match mask buffer.
4900  * @param match_value
4901  *   pointer to match value buffer.
4902  *
4903  * @return
4904  *   0 if valid, -EINVAL otherwise.
4905  */
4906 static int
4907 flow_dv_check_valid_spec(void *match_mask, void *match_value)
4908 {
4909         uint8_t *m = match_mask;
4910         uint8_t *v = match_value;
4911         unsigned int i;
4912
4913         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
4914                 if (v[i] & ~m[i]) {
4915                         DRV_LOG(ERR,
4916                                 "match_value differs from match_criteria"
4917                                 " %p[%u] != %p[%u]",
4918                                 match_value, i, match_mask, i);
4919                         return -EINVAL;
4920                 }
4921         }
4922         return 0;
4923 }
4924 #endif
4925
4926 /**
4927  * Add Ethernet item to matcher and to the value.
4928  *
4929  * @param[in, out] matcher
4930  *   Flow matcher.
4931  * @param[in, out] key
4932  *   Flow matcher value.
4933  * @param[in] item
4934  *   Flow pattern to translate.
4935  * @param[in] inner
4936  *   Item is inner pattern.
4937  */
4938 static void
4939 flow_dv_translate_item_eth(void *matcher, void *key,
4940                            const struct rte_flow_item *item, int inner)
4941 {
4942         const struct rte_flow_item_eth *eth_m = item->mask;
4943         const struct rte_flow_item_eth *eth_v = item->spec;
4944         const struct rte_flow_item_eth nic_mask = {
4945                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
4946                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
4947                 .type = RTE_BE16(0xffff),
4948         };
4949         void *headers_m;
4950         void *headers_v;
4951         char *l24_v;
4952         unsigned int i;
4953
4954         if (!eth_v)
4955                 return;
4956         if (!eth_m)
4957                 eth_m = &nic_mask;
4958         if (inner) {
4959                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4960                                          inner_headers);
4961                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4962         } else {
4963                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4964                                          outer_headers);
4965                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4966         }
4967         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
4968                &eth_m->dst, sizeof(eth_m->dst));
4969         /* The value must be in the range of the mask. */
4970         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
4971         for (i = 0; i < sizeof(eth_m->dst); ++i)
4972                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
4973         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
4974                &eth_m->src, sizeof(eth_m->src));
4975         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
4976         /* The value must be in the range of the mask. */
4977         for (i = 0; i < sizeof(eth_m->dst); ++i)
4978                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
4979         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
4980                  rte_be_to_cpu_16(eth_m->type));
4981         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
4982         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
4983 }
4984
4985 /**
4986  * Add VLAN item to matcher and to the value.
4987  *
4988  * @param[in, out] dev_flow
4989  *   Flow descriptor.
4990  * @param[in, out] matcher
4991  *   Flow matcher.
4992  * @param[in, out] key
4993  *   Flow matcher value.
4994  * @param[in] item
4995  *   Flow pattern to translate.
4996  * @param[in] inner
4997  *   Item is inner pattern.
4998  */
4999 static void
5000 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5001                             void *matcher, void *key,
5002                             const struct rte_flow_item *item,
5003                             int inner)
5004 {
5005         const struct rte_flow_item_vlan *vlan_m = item->mask;
5006         const struct rte_flow_item_vlan *vlan_v = item->spec;
5007         void *headers_m;
5008         void *headers_v;
5009         uint16_t tci_m;
5010         uint16_t tci_v;
5011
5012         if (!vlan_v)
5013                 return;
5014         if (!vlan_m)
5015                 vlan_m = &rte_flow_item_vlan_mask;
5016         if (inner) {
5017                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5018                                          inner_headers);
5019                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5020         } else {
5021                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5022                                          outer_headers);
5023                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5024                 /*
5025                  * This is workaround, masks are not supported,
5026                  * and pre-validated.
5027                  */
5028                 dev_flow->dv.vf_vlan.tag =
5029                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5030         }
5031         tci_m = rte_be_to_cpu_16(vlan_m->tci);
5032         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5033         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5034         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5035         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5036         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5037         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5038         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5039         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
5040         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
5041         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5042                  rte_be_to_cpu_16(vlan_m->inner_type));
5043         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
5044                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
5045 }
5046
5047 /**
5048  * Add IPV4 item to matcher and to the value.
5049  *
5050  * @param[in, out] matcher
5051  *   Flow matcher.
5052  * @param[in, out] key
5053  *   Flow matcher value.
5054  * @param[in] item
5055  *   Flow pattern to translate.
5056  * @param[in] inner
5057  *   Item is inner pattern.
5058  * @param[in] group
5059  *   The group to insert the rule.
5060  */
5061 static void
5062 flow_dv_translate_item_ipv4(void *matcher, void *key,
5063                             const struct rte_flow_item *item,
5064                             int inner, uint32_t group)
5065 {
5066         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
5067         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
5068         const struct rte_flow_item_ipv4 nic_mask = {
5069                 .hdr = {
5070                         .src_addr = RTE_BE32(0xffffffff),
5071                         .dst_addr = RTE_BE32(0xffffffff),
5072                         .type_of_service = 0xff,
5073                         .next_proto_id = 0xff,
5074                 },
5075         };
5076         void *headers_m;
5077         void *headers_v;
5078         char *l24_m;
5079         char *l24_v;
5080         uint8_t tos;
5081
5082         if (inner) {
5083                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5084                                          inner_headers);
5085                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5086         } else {
5087                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5088                                          outer_headers);
5089                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5090         }
5091         if (group == 0)
5092                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5093         else
5094                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
5095         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
5096         if (!ipv4_v)
5097                 return;
5098         if (!ipv4_m)
5099                 ipv4_m = &nic_mask;
5100         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
5101                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
5102         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5103                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
5104         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
5105         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
5106         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
5107                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
5108         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5109                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
5110         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
5111         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
5112         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
5113         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
5114                  ipv4_m->hdr.type_of_service);
5115         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
5116         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
5117                  ipv4_m->hdr.type_of_service >> 2);
5118         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
5119         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
5120                  ipv4_m->hdr.next_proto_id);
5121         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
5122                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
5123 }
5124
5125 /**
5126  * Add IPV6 item to matcher and to the value.
5127  *
5128  * @param[in, out] matcher
5129  *   Flow matcher.
5130  * @param[in, out] key
5131  *   Flow matcher value.
5132  * @param[in] item
5133  *   Flow pattern to translate.
5134  * @param[in] inner
5135  *   Item is inner pattern.
5136  * @param[in] group
5137  *   The group to insert the rule.
5138  */
5139 static void
5140 flow_dv_translate_item_ipv6(void *matcher, void *key,
5141                             const struct rte_flow_item *item,
5142                             int inner, uint32_t group)
5143 {
5144         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
5145         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
5146         const struct rte_flow_item_ipv6 nic_mask = {
5147                 .hdr = {
5148                         .src_addr =
5149                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
5150                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
5151                         .dst_addr =
5152                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
5153                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
5154                         .vtc_flow = RTE_BE32(0xffffffff),
5155                         .proto = 0xff,
5156                         .hop_limits = 0xff,
5157                 },
5158         };
5159         void *headers_m;
5160         void *headers_v;
5161         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5162         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5163         char *l24_m;
5164         char *l24_v;
5165         uint32_t vtc_m;
5166         uint32_t vtc_v;
5167         int i;
5168         int size;
5169
5170         if (inner) {
5171                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5172                                          inner_headers);
5173                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5174         } else {
5175                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5176                                          outer_headers);
5177                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5178         }
5179         if (group == 0)
5180                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5181         else
5182                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
5183         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
5184         if (!ipv6_v)
5185                 return;
5186         if (!ipv6_m)
5187                 ipv6_m = &nic_mask;
5188         size = sizeof(ipv6_m->hdr.dst_addr);
5189         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
5190                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
5191         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5192                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
5193         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
5194         for (i = 0; i < size; ++i)
5195                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
5196         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
5197                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
5198         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5199                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
5200         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
5201         for (i = 0; i < size; ++i)
5202                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
5203         /* TOS. */
5204         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
5205         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
5206         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
5207         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
5208         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
5209         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
5210         /* Label. */
5211         if (inner) {
5212                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
5213                          vtc_m);
5214                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
5215                          vtc_v);
5216         } else {
5217                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
5218                          vtc_m);
5219                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
5220                          vtc_v);
5221         }
5222         /* Protocol. */
5223         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
5224                  ipv6_m->hdr.proto);
5225         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
5226                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
5227 }
5228
5229 /**
5230  * Add TCP item to matcher and to the value.
5231  *
5232  * @param[in, out] matcher
5233  *   Flow matcher.
5234  * @param[in, out] key
5235  *   Flow matcher value.
5236  * @param[in] item
5237  *   Flow pattern to translate.
5238  * @param[in] inner
5239  *   Item is inner pattern.
5240  */
5241 static void
5242 flow_dv_translate_item_tcp(void *matcher, void *key,
5243                            const struct rte_flow_item *item,
5244                            int inner)
5245 {
5246         const struct rte_flow_item_tcp *tcp_m = item->mask;
5247         const struct rte_flow_item_tcp *tcp_v = item->spec;
5248         void *headers_m;
5249         void *headers_v;
5250
5251         if (inner) {
5252                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5253                                          inner_headers);
5254                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5255         } else {
5256                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5257                                          outer_headers);
5258                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5259         }
5260         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
5261         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
5262         if (!tcp_v)
5263                 return;
5264         if (!tcp_m)
5265                 tcp_m = &rte_flow_item_tcp_mask;
5266         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
5267                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
5268         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
5269                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
5270         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
5271                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
5272         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
5273                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
5274         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
5275                  tcp_m->hdr.tcp_flags);
5276         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
5277                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
5278 }
5279
5280 /**
5281  * Add UDP item to matcher and to the value.
5282  *
5283  * @param[in, out] matcher
5284  *   Flow matcher.
5285  * @param[in, out] key
5286  *   Flow matcher value.
5287  * @param[in] item
5288  *   Flow pattern to translate.
5289  * @param[in] inner
5290  *   Item is inner pattern.
5291  */
5292 static void
5293 flow_dv_translate_item_udp(void *matcher, void *key,
5294                            const struct rte_flow_item *item,
5295                            int inner)
5296 {
5297         const struct rte_flow_item_udp *udp_m = item->mask;
5298         const struct rte_flow_item_udp *udp_v = item->spec;
5299         void *headers_m;
5300         void *headers_v;
5301
5302         if (inner) {
5303                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5304                                          inner_headers);
5305                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5306         } else {
5307                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5308                                          outer_headers);
5309                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5310         }
5311         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
5312         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
5313         if (!udp_v)
5314                 return;
5315         if (!udp_m)
5316                 udp_m = &rte_flow_item_udp_mask;
5317         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
5318                  rte_be_to_cpu_16(udp_m->hdr.src_port));
5319         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
5320                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
5321         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
5322                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
5323         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
5324                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
5325 }
5326
5327 /**
5328  * Add GRE optional Key item to matcher and to the value.
5329  *
5330  * @param[in, out] matcher
5331  *   Flow matcher.
5332  * @param[in, out] key
5333  *   Flow matcher value.
5334  * @param[in] item
5335  *   Flow pattern to translate.
5336  * @param[in] inner
5337  *   Item is inner pattern.
5338  */
5339 static void
5340 flow_dv_translate_item_gre_key(void *matcher, void *key,
5341                                    const struct rte_flow_item *item)
5342 {
5343         const rte_be32_t *key_m = item->mask;
5344         const rte_be32_t *key_v = item->spec;
5345         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5346         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5347         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
5348
5349         if (!key_v)
5350                 return;
5351         if (!key_m)
5352                 key_m = &gre_key_default_mask;
5353         /* GRE K bit must be on and should already be validated */
5354         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
5355         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
5356         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
5357                  rte_be_to_cpu_32(*key_m) >> 8);
5358         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
5359                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
5360         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
5361                  rte_be_to_cpu_32(*key_m) & 0xFF);
5362         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
5363                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
5364 }
5365
5366 /**
5367  * Add GRE item to matcher and to the value.
5368  *
5369  * @param[in, out] matcher
5370  *   Flow matcher.
5371  * @param[in, out] key
5372  *   Flow matcher value.
5373  * @param[in] item
5374  *   Flow pattern to translate.
5375  * @param[in] inner
5376  *   Item is inner pattern.
5377  */
5378 static void
5379 flow_dv_translate_item_gre(void *matcher, void *key,
5380                            const struct rte_flow_item *item,
5381                            int inner)
5382 {
5383         const struct rte_flow_item_gre *gre_m = item->mask;
5384         const struct rte_flow_item_gre *gre_v = item->spec;
5385         void *headers_m;
5386         void *headers_v;
5387         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5388         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5389         struct {
5390                 union {
5391                         __extension__
5392                         struct {
5393                                 uint16_t version:3;
5394                                 uint16_t rsvd0:9;
5395                                 uint16_t s_present:1;
5396                                 uint16_t k_present:1;
5397                                 uint16_t rsvd_bit1:1;
5398                                 uint16_t c_present:1;
5399                         };
5400                         uint16_t value;
5401                 };
5402         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
5403
5404         if (inner) {
5405                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5406                                          inner_headers);
5407                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5408         } else {
5409                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5410                                          outer_headers);
5411                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5412         }
5413         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
5414         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
5415         if (!gre_v)
5416                 return;
5417         if (!gre_m)
5418                 gre_m = &rte_flow_item_gre_mask;
5419         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
5420                  rte_be_to_cpu_16(gre_m->protocol));
5421         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
5422                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
5423         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
5424         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
5425         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
5426                  gre_crks_rsvd0_ver_m.c_present);
5427         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
5428                  gre_crks_rsvd0_ver_v.c_present &
5429                  gre_crks_rsvd0_ver_m.c_present);
5430         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
5431                  gre_crks_rsvd0_ver_m.k_present);
5432         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
5433                  gre_crks_rsvd0_ver_v.k_present &
5434                  gre_crks_rsvd0_ver_m.k_present);
5435         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
5436                  gre_crks_rsvd0_ver_m.s_present);
5437         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
5438                  gre_crks_rsvd0_ver_v.s_present &
5439                  gre_crks_rsvd0_ver_m.s_present);
5440 }
5441
5442 /**
5443  * Add NVGRE item to matcher and to the value.
5444  *
5445  * @param[in, out] matcher
5446  *   Flow matcher.
5447  * @param[in, out] key
5448  *   Flow matcher value.
5449  * @param[in] item
5450  *   Flow pattern to translate.
5451  * @param[in] inner
5452  *   Item is inner pattern.
5453  */
5454 static void
5455 flow_dv_translate_item_nvgre(void *matcher, void *key,
5456                              const struct rte_flow_item *item,
5457                              int inner)
5458 {
5459         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
5460         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
5461         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5462         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5463         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
5464         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
5465         char *gre_key_m;
5466         char *gre_key_v;
5467         int size;
5468         int i;
5469
5470         /* For NVGRE, GRE header fields must be set with defined values. */
5471         const struct rte_flow_item_gre gre_spec = {
5472                 .c_rsvd0_ver = RTE_BE16(0x2000),
5473                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
5474         };
5475         const struct rte_flow_item_gre gre_mask = {
5476                 .c_rsvd0_ver = RTE_BE16(0xB000),
5477                 .protocol = RTE_BE16(UINT16_MAX),
5478         };
5479         const struct rte_flow_item gre_item = {
5480                 .spec = &gre_spec,
5481                 .mask = &gre_mask,
5482                 .last = NULL,
5483         };
5484         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
5485         if (!nvgre_v)
5486                 return;
5487         if (!nvgre_m)
5488                 nvgre_m = &rte_flow_item_nvgre_mask;
5489         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
5490         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
5491         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
5492         memcpy(gre_key_m, tni_flow_id_m, size);
5493         for (i = 0; i < size; ++i)
5494                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
5495 }
5496
5497 /**
5498  * Add VXLAN item to matcher and to the value.
5499  *
5500  * @param[in, out] matcher
5501  *   Flow matcher.
5502  * @param[in, out] key
5503  *   Flow matcher value.
5504  * @param[in] item
5505  *   Flow pattern to translate.
5506  * @param[in] inner
5507  *   Item is inner pattern.
5508  */
5509 static void
5510 flow_dv_translate_item_vxlan(void *matcher, void *key,
5511                              const struct rte_flow_item *item,
5512                              int inner)
5513 {
5514         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
5515         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
5516         void *headers_m;
5517         void *headers_v;
5518         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5519         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5520         char *vni_m;
5521         char *vni_v;
5522         uint16_t dport;
5523         int size;
5524         int i;
5525
5526         if (inner) {
5527                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5528                                          inner_headers);
5529                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5530         } else {
5531                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5532                                          outer_headers);
5533                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5534         }
5535         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
5536                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
5537         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
5538                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
5539                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
5540         }
5541         if (!vxlan_v)
5542                 return;
5543         if (!vxlan_m)
5544                 vxlan_m = &rte_flow_item_vxlan_mask;
5545         size = sizeof(vxlan_m->vni);
5546         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
5547         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
5548         memcpy(vni_m, vxlan_m->vni, size);
5549         for (i = 0; i < size; ++i)
5550                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
5551 }
5552
5553 /**
5554  * Add Geneve item to matcher and to the value.
5555  *
5556  * @param[in, out] matcher
5557  *   Flow matcher.
5558  * @param[in, out] key
5559  *   Flow matcher value.
5560  * @param[in] item
5561  *   Flow pattern to translate.
5562  * @param[in] inner
5563  *   Item is inner pattern.
5564  */
5565
5566 static void
5567 flow_dv_translate_item_geneve(void *matcher, void *key,
5568                               const struct rte_flow_item *item, int inner)
5569 {
5570         const struct rte_flow_item_geneve *geneve_m = item->mask;
5571         const struct rte_flow_item_geneve *geneve_v = item->spec;
5572         void *headers_m;
5573         void *headers_v;
5574         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5575         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5576         uint16_t dport;
5577         uint16_t gbhdr_m;
5578         uint16_t gbhdr_v;
5579         char *vni_m;
5580         char *vni_v;
5581         size_t size, i;
5582
5583         if (inner) {
5584                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5585                                          inner_headers);
5586                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5587         } else {
5588                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5589                                          outer_headers);
5590                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5591         }
5592         dport = MLX5_UDP_PORT_GENEVE;
5593         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
5594                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
5595                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
5596         }
5597         if (!geneve_v)
5598                 return;
5599         if (!geneve_m)
5600                 geneve_m = &rte_flow_item_geneve_mask;
5601         size = sizeof(geneve_m->vni);
5602         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
5603         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
5604         memcpy(vni_m, geneve_m->vni, size);
5605         for (i = 0; i < size; ++i)
5606                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
5607         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
5608                  rte_be_to_cpu_16(geneve_m->protocol));
5609         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
5610                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
5611         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
5612         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
5613         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
5614                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
5615         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
5616                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
5617         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
5618                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
5619         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
5620                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
5621                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
5622 }
5623
5624 /**
5625  * Add MPLS item to matcher and to the value.
5626  *
5627  * @param[in, out] matcher
5628  *   Flow matcher.
5629  * @param[in, out] key
5630  *   Flow matcher value.
5631  * @param[in] item
5632  *   Flow pattern to translate.
5633  * @param[in] prev_layer
5634  *   The protocol layer indicated in previous item.
5635  * @param[in] inner
5636  *   Item is inner pattern.
5637  */
5638 static void
5639 flow_dv_translate_item_mpls(void *matcher, void *key,
5640                             const struct rte_flow_item *item,
5641                             uint64_t prev_layer,
5642                             int inner)
5643 {
5644         const uint32_t *in_mpls_m = item->mask;
5645         const uint32_t *in_mpls_v = item->spec;
5646         uint32_t *out_mpls_m = 0;
5647         uint32_t *out_mpls_v = 0;
5648         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5649         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5650         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
5651                                      misc_parameters_2);
5652         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
5653         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
5654         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5655
5656         switch (prev_layer) {
5657         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
5658                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
5659                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
5660                          MLX5_UDP_PORT_MPLS);
5661                 break;
5662         case MLX5_FLOW_LAYER_GRE:
5663                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
5664                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
5665                          RTE_ETHER_TYPE_MPLS);
5666                 break;
5667         default:
5668                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
5669                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
5670                          IPPROTO_MPLS);
5671                 break;
5672         }
5673         if (!in_mpls_v)
5674                 return;
5675         if (!in_mpls_m)
5676                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
5677         switch (prev_layer) {
5678         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
5679                 out_mpls_m =
5680                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
5681                                                  outer_first_mpls_over_udp);
5682                 out_mpls_v =
5683                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
5684                                                  outer_first_mpls_over_udp);
5685                 break;
5686         case MLX5_FLOW_LAYER_GRE:
5687                 out_mpls_m =
5688                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
5689                                                  outer_first_mpls_over_gre);
5690                 out_mpls_v =
5691                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
5692                                                  outer_first_mpls_over_gre);
5693                 break;
5694         default:
5695                 /* Inner MPLS not over GRE is not supported. */
5696                 if (!inner) {
5697                         out_mpls_m =
5698                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
5699                                                          misc2_m,
5700                                                          outer_first_mpls);
5701                         out_mpls_v =
5702                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
5703                                                          misc2_v,
5704                                                          outer_first_mpls);
5705                 }
5706                 break;
5707         }
5708         if (out_mpls_m && out_mpls_v) {
5709                 *out_mpls_m = *in_mpls_m;
5710                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
5711         }
5712 }
5713
5714 /**
5715  * Add metadata register item to matcher
5716  *
5717  * @param[in, out] matcher
5718  *   Flow matcher.
5719  * @param[in, out] key
5720  *   Flow matcher value.
5721  * @param[in] reg_type
5722  *   Type of device metadata register
5723  * @param[in] value
5724  *   Register value
5725  * @param[in] mask
5726  *   Register mask
5727  */
5728 static void
5729 flow_dv_match_meta_reg(void *matcher, void *key,
5730                        enum modify_reg reg_type,
5731                        uint32_t data, uint32_t mask)
5732 {
5733         void *misc2_m =
5734                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
5735         void *misc2_v =
5736                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
5737
5738         data &= mask;
5739         switch (reg_type) {
5740         case REG_A:
5741                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
5742                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
5743                 break;
5744         case REG_B:
5745                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
5746                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
5747                 break;
5748         case REG_C_0:
5749                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, mask);
5750                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, data);
5751                 break;
5752         case REG_C_1:
5753                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
5754                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
5755                 break;
5756         case REG_C_2:
5757                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
5758                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
5759                 break;
5760         case REG_C_3:
5761                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
5762                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
5763                 break;
5764         case REG_C_4:
5765                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
5766                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
5767                 break;
5768         case REG_C_5:
5769                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
5770                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
5771                 break;
5772         case REG_C_6:
5773                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
5774                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
5775                 break;
5776         case REG_C_7:
5777                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
5778                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
5779                 break;
5780         default:
5781                 assert(false);
5782                 break;
5783         }
5784 }
5785
5786 /**
5787  * Add MARK item to matcher
5788  *
5789  * @param[in] dev
5790  *   The device to configure through.
5791  * @param[in, out] matcher
5792  *   Flow matcher.
5793  * @param[in, out] key
5794  *   Flow matcher value.
5795  * @param[in] item
5796  *   Flow pattern to translate.
5797  */
5798 static void
5799 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
5800                             void *matcher, void *key,
5801                             const struct rte_flow_item *item)
5802 {
5803         struct mlx5_priv *priv = dev->data->dev_private;
5804         const struct rte_flow_item_mark *mark;
5805         uint32_t value;
5806         uint32_t mask;
5807
5808         mark = item->mask ? (const void *)item->mask :
5809                             &rte_flow_item_mark_mask;
5810         mask = mark->id & priv->sh->dv_mark_mask;
5811         mark = (const void *)item->spec;
5812         assert(mark);
5813         value = mark->id & priv->sh->dv_mark_mask & mask;
5814         if (mask) {
5815                 enum modify_reg reg;
5816
5817                 /* Get the metadata register index for the mark. */
5818                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
5819                 assert(reg > 0);
5820                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
5821         }
5822 }
5823
5824 /**
5825  * Add META item to matcher
5826  *
5827  * @param[in] dev
5828  *   The devich to configure through.
5829  * @param[in, out] matcher
5830  *   Flow matcher.
5831  * @param[in, out] key
5832  *   Flow matcher value.
5833  * @param[in] attr
5834  *   Attributes of flow that includes this item.
5835  * @param[in] item
5836  *   Flow pattern to translate.
5837  */
5838 static void
5839 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
5840                             void *matcher, void *key,
5841                             const struct rte_flow_attr *attr,
5842                             const struct rte_flow_item *item)
5843 {
5844         const struct rte_flow_item_meta *meta_m;
5845         const struct rte_flow_item_meta *meta_v;
5846
5847         meta_m = (const void *)item->mask;
5848         if (!meta_m)
5849                 meta_m = &rte_flow_item_meta_mask;
5850         meta_v = (const void *)item->spec;
5851         if (meta_v) {
5852                 enum modify_reg reg;
5853                 uint32_t value = meta_v->data;
5854                 uint32_t mask = meta_m->data;
5855
5856                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
5857                 if (reg < 0)
5858                         return;
5859                 /*
5860                  * In datapath code there is no endianness
5861                  * coversions for perfromance reasons, all
5862                  * pattern conversions are done in rte_flow.
5863                  */
5864                 value = rte_cpu_to_be_32(value);
5865                 mask = rte_cpu_to_be_32(mask);
5866                 if (reg == REG_C_0) {
5867                         struct mlx5_priv *priv = dev->data->dev_private;
5868                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
5869                         uint32_t shl_c0 = rte_bsf32(msk_c0);
5870
5871                         msk_c0 = rte_cpu_to_be_32(msk_c0);
5872                         value <<= shl_c0;
5873                         mask <<= shl_c0;
5874                         assert(msk_c0);
5875                         assert(!(~msk_c0 & mask));
5876                 }
5877                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
5878         }
5879 }
5880
5881 /**
5882  * Add vport metadata Reg C0 item to matcher
5883  *
5884  * @param[in, out] matcher
5885  *   Flow matcher.
5886  * @param[in, out] key
5887  *   Flow matcher value.
5888  * @param[in] reg
5889  *   Flow pattern to translate.
5890  */
5891 static void
5892 flow_dv_translate_item_meta_vport(void *matcher, void *key,
5893                                   uint32_t value, uint32_t mask)
5894 {
5895         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
5896 }
5897
5898 /**
5899  * Add tag item to matcher
5900  *
5901  * @param[in, out] matcher
5902  *   Flow matcher.
5903  * @param[in, out] key
5904  *   Flow matcher value.
5905  * @param[in] item
5906  *   Flow pattern to translate.
5907  */
5908 static void
5909 flow_dv_translate_mlx5_item_tag(void *matcher, void *key,
5910                                 const struct rte_flow_item *item)
5911 {
5912         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
5913         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
5914
5915         assert(tag_v);
5916         flow_dv_match_meta_reg(matcher, key, tag_v->id, tag_v->data,
5917                                tag_m ? tag_m->data : UINT32_MAX);
5918 }
5919
5920 /**
5921  * Add TAG item to matcher
5922  *
5923  * @param[in] dev
5924  *   The devich to configure through.
5925  * @param[in, out] matcher
5926  *   Flow matcher.
5927  * @param[in, out] key
5928  *   Flow matcher value.
5929  * @param[in] item
5930  *   Flow pattern to translate.
5931  */
5932 static void
5933 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
5934                            void *matcher, void *key,
5935                            const struct rte_flow_item *item)
5936 {
5937         const struct rte_flow_item_tag *tag_v = item->spec;
5938         const struct rte_flow_item_tag *tag_m = item->mask;
5939         enum modify_reg reg;
5940
5941         assert(tag_v);
5942         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
5943         /* Get the metadata register index for the tag. */
5944         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
5945         assert(reg > 0);
5946         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
5947 }
5948
5949 /**
5950  * Add source vport match to the specified matcher.
5951  *
5952  * @param[in, out] matcher
5953  *   Flow matcher.
5954  * @param[in, out] key
5955  *   Flow matcher value.
5956  * @param[in] port
5957  *   Source vport value to match
5958  * @param[in] mask
5959  *   Mask
5960  */
5961 static void
5962 flow_dv_translate_item_source_vport(void *matcher, void *key,
5963                                     int16_t port, uint16_t mask)
5964 {
5965         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5966         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5967
5968         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
5969         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
5970 }
5971
5972 /**
5973  * Translate port-id item to eswitch match on  port-id.
5974  *
5975  * @param[in] dev
5976  *   The devich to configure through.
5977  * @param[in, out] matcher
5978  *   Flow matcher.
5979  * @param[in, out] key
5980  *   Flow matcher value.
5981  * @param[in] item
5982  *   Flow pattern to translate.
5983  *
5984  * @return
5985  *   0 on success, a negative errno value otherwise.
5986  */
5987 static int
5988 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
5989                                void *key, const struct rte_flow_item *item)
5990 {
5991         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
5992         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
5993         struct mlx5_priv *priv;
5994         uint16_t mask, id;
5995
5996         mask = pid_m ? pid_m->id : 0xffff;
5997         id = pid_v ? pid_v->id : dev->data->port_id;
5998         priv = mlx5_port_to_eswitch_info(id, item == NULL);
5999         if (!priv)
6000                 return -rte_errno;
6001         /* Translate to vport field or to metadata, depending on mode. */
6002         if (priv->vport_meta_mask)
6003                 flow_dv_translate_item_meta_vport(matcher, key,
6004                                                   priv->vport_meta_tag,
6005                                                   priv->vport_meta_mask);
6006         else
6007                 flow_dv_translate_item_source_vport(matcher, key,
6008                                                     priv->vport_id, mask);
6009         return 0;
6010 }
6011
6012 /**
6013  * Add ICMP6 item to matcher and to the value.
6014  *
6015  * @param[in, out] matcher
6016  *   Flow matcher.
6017  * @param[in, out] key
6018  *   Flow matcher value.
6019  * @param[in] item
6020  *   Flow pattern to translate.
6021  * @param[in] inner
6022  *   Item is inner pattern.
6023  */
6024 static void
6025 flow_dv_translate_item_icmp6(void *matcher, void *key,
6026                               const struct rte_flow_item *item,
6027                               int inner)
6028 {
6029         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
6030         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
6031         void *headers_m;
6032         void *headers_v;
6033         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
6034                                      misc_parameters_3);
6035         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6036         if (inner) {
6037                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6038                                          inner_headers);
6039                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6040         } else {
6041                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6042                                          outer_headers);
6043                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6044         }
6045         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
6046         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
6047         if (!icmp6_v)
6048                 return;
6049         if (!icmp6_m)
6050                 icmp6_m = &rte_flow_item_icmp6_mask;
6051         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
6052         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
6053                  icmp6_v->type & icmp6_m->type);
6054         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
6055         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
6056                  icmp6_v->code & icmp6_m->code);
6057 }
6058
6059 /**
6060  * Add ICMP item to matcher and to the value.
6061  *
6062  * @param[in, out] matcher
6063  *   Flow matcher.
6064  * @param[in, out] key
6065  *   Flow matcher value.
6066  * @param[in] item
6067  *   Flow pattern to translate.
6068  * @param[in] inner
6069  *   Item is inner pattern.
6070  */
6071 static void
6072 flow_dv_translate_item_icmp(void *matcher, void *key,
6073                             const struct rte_flow_item *item,
6074                             int inner)
6075 {
6076         const struct rte_flow_item_icmp *icmp_m = item->mask;
6077         const struct rte_flow_item_icmp *icmp_v = item->spec;
6078         void *headers_m;
6079         void *headers_v;
6080         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
6081                                      misc_parameters_3);
6082         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6083         if (inner) {
6084                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6085                                          inner_headers);
6086                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6087         } else {
6088                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6089                                          outer_headers);
6090                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6091         }
6092         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
6093         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
6094         if (!icmp_v)
6095                 return;
6096         if (!icmp_m)
6097                 icmp_m = &rte_flow_item_icmp_mask;
6098         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
6099                  icmp_m->hdr.icmp_type);
6100         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
6101                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
6102         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
6103                  icmp_m->hdr.icmp_code);
6104         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
6105                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
6106 }
6107
6108 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
6109
6110 #define HEADER_IS_ZERO(match_criteria, headers)                              \
6111         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
6112                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
6113
6114 /**
6115  * Calculate flow matcher enable bitmap.
6116  *
6117  * @param match_criteria
6118  *   Pointer to flow matcher criteria.
6119  *
6120  * @return
6121  *   Bitmap of enabled fields.
6122  */
6123 static uint8_t
6124 flow_dv_matcher_enable(uint32_t *match_criteria)
6125 {
6126         uint8_t match_criteria_enable;
6127
6128         match_criteria_enable =
6129                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
6130                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
6131         match_criteria_enable |=
6132                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
6133                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
6134         match_criteria_enable |=
6135                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
6136                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
6137         match_criteria_enable |=
6138                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
6139                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
6140         match_criteria_enable |=
6141                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
6142                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
6143         return match_criteria_enable;
6144 }
6145
6146
6147 /**
6148  * Get a flow table.
6149  *
6150  * @param[in, out] dev
6151  *   Pointer to rte_eth_dev structure.
6152  * @param[in] table_id
6153  *   Table id to use.
6154  * @param[in] egress
6155  *   Direction of the table.
6156  * @param[in] transfer
6157  *   E-Switch or NIC flow.
6158  * @param[out] error
6159  *   pointer to error structure.
6160  *
6161  * @return
6162  *   Returns tables resource based on the index, NULL in case of failed.
6163  */
6164 static struct mlx5_flow_tbl_resource *
6165 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
6166                          uint32_t table_id, uint8_t egress,
6167                          uint8_t transfer,
6168                          struct rte_flow_error *error)
6169 {
6170         struct mlx5_priv *priv = dev->data->dev_private;
6171         struct mlx5_ibv_shared *sh = priv->sh;
6172         struct mlx5_flow_tbl_resource *tbl;
6173         union mlx5_flow_tbl_key table_key = {
6174                 {
6175                         .table_id = table_id,
6176                         .reserved = 0,
6177                         .domain = !!transfer,
6178                         .direction = !!egress,
6179                 }
6180         };
6181         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
6182                                                          table_key.v64);
6183         struct mlx5_flow_tbl_data_entry *tbl_data;
6184         int ret;
6185         void *domain;
6186
6187         if (pos) {
6188                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
6189                                         entry);
6190                 tbl = &tbl_data->tbl;
6191                 rte_atomic32_inc(&tbl->refcnt);
6192                 return tbl;
6193         }
6194         tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0);
6195         if (!tbl_data) {
6196                 rte_flow_error_set(error, ENOMEM,
6197                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6198                                    NULL,
6199                                    "cannot allocate flow table data entry");
6200                 return NULL;
6201         }
6202         tbl = &tbl_data->tbl;
6203         pos = &tbl_data->entry;
6204         if (transfer)
6205                 domain = sh->fdb_domain;
6206         else if (egress)
6207                 domain = sh->tx_domain;
6208         else
6209                 domain = sh->rx_domain;
6210         tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id);
6211         if (!tbl->obj) {
6212                 rte_flow_error_set(error, ENOMEM,
6213                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6214                                    NULL, "cannot create flow table object");
6215                 rte_free(tbl_data);
6216                 return NULL;
6217         }
6218         /*
6219          * No multi-threads now, but still better to initialize the reference
6220          * count before insert it into the hash list.
6221          */
6222         rte_atomic32_init(&tbl->refcnt);
6223         /* Jump action reference count is initialized here. */
6224         rte_atomic32_init(&tbl_data->jump.refcnt);
6225         pos->key = table_key.v64;
6226         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
6227         if (ret < 0) {
6228                 rte_flow_error_set(error, -ret,
6229                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6230                                    "cannot insert flow table data entry");
6231                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
6232                 rte_free(tbl_data);
6233         }
6234         rte_atomic32_inc(&tbl->refcnt);
6235         return tbl;
6236 }
6237
6238 /**
6239  * Release a flow table.
6240  *
6241  * @param[in] dev
6242  *   Pointer to rte_eth_dev structure.
6243  * @param[in] tbl
6244  *   Table resource to be released.
6245  *
6246  * @return
6247  *   Returns 0 if table was released, else return 1;
6248  */
6249 static int
6250 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
6251                              struct mlx5_flow_tbl_resource *tbl)
6252 {
6253         struct mlx5_priv *priv = dev->data->dev_private;
6254         struct mlx5_ibv_shared *sh = priv->sh;
6255         struct mlx5_flow_tbl_data_entry *tbl_data =
6256                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
6257
6258         if (!tbl)
6259                 return 0;
6260         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
6261                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
6262
6263                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
6264                 tbl->obj = NULL;
6265                 /* remove the entry from the hash list and free memory. */
6266                 mlx5_hlist_remove(sh->flow_tbls, pos);
6267                 rte_free(tbl_data);
6268                 return 0;
6269         }
6270         return 1;
6271 }
6272
6273 /**
6274  * Register the flow matcher.
6275  *
6276  * @param[in, out] dev
6277  *   Pointer to rte_eth_dev structure.
6278  * @param[in, out] matcher
6279  *   Pointer to flow matcher.
6280  * @param[in, out] key
6281  *   Pointer to flow table key.
6282  * @parm[in, out] dev_flow
6283  *   Pointer to the dev_flow.
6284  * @param[out] error
6285  *   pointer to error structure.
6286  *
6287  * @return
6288  *   0 on success otherwise -errno and errno is set.
6289  */
6290 static int
6291 flow_dv_matcher_register(struct rte_eth_dev *dev,
6292                          struct mlx5_flow_dv_matcher *matcher,
6293                          union mlx5_flow_tbl_key *key,
6294                          struct mlx5_flow *dev_flow,
6295                          struct rte_flow_error *error)
6296 {
6297         struct mlx5_priv *priv = dev->data->dev_private;
6298         struct mlx5_ibv_shared *sh = priv->sh;
6299         struct mlx5_flow_dv_matcher *cache_matcher;
6300         struct mlx5dv_flow_matcher_attr dv_attr = {
6301                 .type = IBV_FLOW_ATTR_NORMAL,
6302                 .match_mask = (void *)&matcher->mask,
6303         };
6304         struct mlx5_flow_tbl_resource *tbl;
6305         struct mlx5_flow_tbl_data_entry *tbl_data;
6306
6307         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
6308                                        key->domain, error);
6309         if (!tbl)
6310                 return -rte_errno;      /* No need to refill the error info */
6311         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
6312         /* Lookup from cache. */
6313         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
6314                 if (matcher->crc == cache_matcher->crc &&
6315                     matcher->priority == cache_matcher->priority &&
6316                     !memcmp((const void *)matcher->mask.buf,
6317                             (const void *)cache_matcher->mask.buf,
6318                             cache_matcher->mask.size)) {
6319                         DRV_LOG(DEBUG,
6320                                 "%s group %u priority %hd use %s "
6321                                 "matcher %p: refcnt %d++",
6322                                 key->domain ? "FDB" : "NIC", key->table_id,
6323                                 cache_matcher->priority,
6324                                 key->direction ? "tx" : "rx",
6325                                 (void *)cache_matcher,
6326                                 rte_atomic32_read(&cache_matcher->refcnt));
6327                         rte_atomic32_inc(&cache_matcher->refcnt);
6328                         dev_flow->dv.matcher = cache_matcher;
6329                         /* old matcher should not make the table ref++. */
6330                         flow_dv_tbl_resource_release(dev, tbl);
6331                         return 0;
6332                 }
6333         }
6334         /* Register new matcher. */
6335         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
6336         if (!cache_matcher) {
6337                 flow_dv_tbl_resource_release(dev, tbl);
6338                 return rte_flow_error_set(error, ENOMEM,
6339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6340                                           "cannot allocate matcher memory");
6341         }
6342         *cache_matcher = *matcher;
6343         dv_attr.match_criteria_enable =
6344                 flow_dv_matcher_enable(cache_matcher->mask.buf);
6345         dv_attr.priority = matcher->priority;
6346         if (key->direction)
6347                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
6348         cache_matcher->matcher_object =
6349                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
6350         if (!cache_matcher->matcher_object) {
6351                 rte_free(cache_matcher);
6352 #ifdef HAVE_MLX5DV_DR
6353                 flow_dv_tbl_resource_release(dev, tbl);
6354 #endif
6355                 return rte_flow_error_set(error, ENOMEM,
6356                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6357                                           NULL, "cannot create matcher");
6358         }
6359         /* Save the table information */
6360         cache_matcher->tbl = tbl;
6361         rte_atomic32_init(&cache_matcher->refcnt);
6362         /* only matcher ref++, table ref++ already done above in get API. */
6363         rte_atomic32_inc(&cache_matcher->refcnt);
6364         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
6365         dev_flow->dv.matcher = cache_matcher;
6366         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
6367                 key->domain ? "FDB" : "NIC", key->table_id,
6368                 cache_matcher->priority,
6369                 key->direction ? "tx" : "rx", (void *)cache_matcher,
6370                 rte_atomic32_read(&cache_matcher->refcnt));
6371         return 0;
6372 }
6373
6374 /**
6375  * Find existing tag resource or create and register a new one.
6376  *
6377  * @param dev[in, out]
6378  *   Pointer to rte_eth_dev structure.
6379  * @param[in, out] tag_be24
6380  *   Tag value in big endian then R-shift 8.
6381  * @parm[in, out] dev_flow
6382  *   Pointer to the dev_flow.
6383  * @param[out] error
6384  *   pointer to error structure.
6385  *
6386  * @return
6387  *   0 on success otherwise -errno and errno is set.
6388  */
6389 static int
6390 flow_dv_tag_resource_register
6391                         (struct rte_eth_dev *dev,
6392                          uint32_t tag_be24,
6393                          struct mlx5_flow *dev_flow,
6394                          struct rte_flow_error *error)
6395 {
6396         struct mlx5_priv *priv = dev->data->dev_private;
6397         struct mlx5_ibv_shared *sh = priv->sh;
6398         struct mlx5_flow_dv_tag_resource *cache_resource;
6399         struct mlx5_hlist_entry *entry;
6400
6401         /* Lookup a matching resource from cache. */
6402         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
6403         if (entry) {
6404                 cache_resource = container_of
6405                         (entry, struct mlx5_flow_dv_tag_resource, entry);
6406                 rte_atomic32_inc(&cache_resource->refcnt);
6407                 dev_flow->dv.tag_resource = cache_resource;
6408                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
6409                         (void *)cache_resource,
6410                         rte_atomic32_read(&cache_resource->refcnt));
6411                 return 0;
6412         }
6413         /* Register new resource. */
6414         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
6415         if (!cache_resource)
6416                 return rte_flow_error_set(error, ENOMEM,
6417                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6418                                           "cannot allocate resource memory");
6419         cache_resource->entry.key = (uint64_t)tag_be24;
6420         cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24);
6421         if (!cache_resource->action) {
6422                 rte_free(cache_resource);
6423                 return rte_flow_error_set(error, ENOMEM,
6424                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6425                                           NULL, "cannot create action");
6426         }
6427         rte_atomic32_init(&cache_resource->refcnt);
6428         rte_atomic32_inc(&cache_resource->refcnt);
6429         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
6430                 mlx5_glue->destroy_flow_action(cache_resource->action);
6431                 rte_free(cache_resource);
6432                 return rte_flow_error_set(error, EEXIST,
6433                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6434                                           NULL, "cannot insert tag");
6435         }
6436         dev_flow->dv.tag_resource = cache_resource;
6437         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
6438                 (void *)cache_resource,
6439                 rte_atomic32_read(&cache_resource->refcnt));
6440         return 0;
6441 }
6442
6443 /**
6444  * Release the tag.
6445  *
6446  * @param dev
6447  *   Pointer to Ethernet device.
6448  * @param flow
6449  *   Pointer to mlx5_flow.
6450  *
6451  * @return
6452  *   1 while a reference on it exists, 0 when freed.
6453  */
6454 static int
6455 flow_dv_tag_release(struct rte_eth_dev *dev,
6456                     struct mlx5_flow_dv_tag_resource *tag)
6457 {
6458         struct mlx5_priv *priv = dev->data->dev_private;
6459         struct mlx5_ibv_shared *sh = priv->sh;
6460
6461         assert(tag);
6462         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
6463                 dev->data->port_id, (void *)tag,
6464                 rte_atomic32_read(&tag->refcnt));
6465         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
6466                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
6467                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
6468                 DRV_LOG(DEBUG, "port %u tag %p: removed",
6469                         dev->data->port_id, (void *)tag);
6470                 rte_free(tag);
6471                 return 0;
6472         }
6473         return 1;
6474 }
6475
6476 /**
6477  * Translate port ID action to vport.
6478  *
6479  * @param[in] dev
6480  *   Pointer to rte_eth_dev structure.
6481  * @param[in] action
6482  *   Pointer to the port ID action.
6483  * @param[out] dst_port_id
6484  *   The target port ID.
6485  * @param[out] error
6486  *   Pointer to the error structure.
6487  *
6488  * @return
6489  *   0 on success, a negative errno value otherwise and rte_errno is set.
6490  */
6491 static int
6492 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
6493                                  const struct rte_flow_action *action,
6494                                  uint32_t *dst_port_id,
6495                                  struct rte_flow_error *error)
6496 {
6497         uint32_t port;
6498         struct mlx5_priv *priv;
6499         const struct rte_flow_action_port_id *conf =
6500                         (const struct rte_flow_action_port_id *)action->conf;
6501
6502         port = conf->original ? dev->data->port_id : conf->id;
6503         priv = mlx5_port_to_eswitch_info(port, false);
6504         if (!priv)
6505                 return rte_flow_error_set(error, -rte_errno,
6506                                           RTE_FLOW_ERROR_TYPE_ACTION,
6507                                           NULL,
6508                                           "No eswitch info was found for port");
6509 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
6510         /*
6511          * This parameter is transferred to
6512          * mlx5dv_dr_action_create_dest_ib_port().
6513          */
6514         *dst_port_id = priv->ibv_port;
6515 #else
6516         /*
6517          * Legacy mode, no LAG configurations is supported.
6518          * This parameter is transferred to
6519          * mlx5dv_dr_action_create_dest_vport().
6520          */
6521         *dst_port_id = priv->vport_id;
6522 #endif
6523         return 0;
6524 }
6525
6526 /**
6527  * Add Tx queue matcher
6528  *
6529  * @param[in] dev
6530  *   Pointer to the dev struct.
6531  * @param[in, out] matcher
6532  *   Flow matcher.
6533  * @param[in, out] key
6534  *   Flow matcher value.
6535  * @param[in] item
6536  *   Flow pattern to translate.
6537  * @param[in] inner
6538  *   Item is inner pattern.
6539  */
6540 static void
6541 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
6542                                 void *matcher, void *key,
6543                                 const struct rte_flow_item *item)
6544 {
6545         const struct mlx5_rte_flow_item_tx_queue *queue_m;
6546         const struct mlx5_rte_flow_item_tx_queue *queue_v;
6547         void *misc_m =
6548                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6549         void *misc_v =
6550                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6551         struct mlx5_txq_ctrl *txq;
6552         uint32_t queue;
6553
6554
6555         queue_m = (const void *)item->mask;
6556         if (!queue_m)
6557                 return;
6558         queue_v = (const void *)item->spec;
6559         if (!queue_v)
6560                 return;
6561         txq = mlx5_txq_get(dev, queue_v->queue);
6562         if (!txq)
6563                 return;
6564         queue = txq->obj->sq->id;
6565         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
6566         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
6567                  queue & queue_m->queue);
6568         mlx5_txq_release(dev, queue_v->queue);
6569 }
6570
6571 /**
6572  * Fill the flow with DV spec, lock free
6573  * (mutex should be acquired by caller).
6574  *
6575  * @param[in] dev
6576  *   Pointer to rte_eth_dev structure.
6577  * @param[in, out] dev_flow
6578  *   Pointer to the sub flow.
6579  * @param[in] attr
6580  *   Pointer to the flow attributes.
6581  * @param[in] items
6582  *   Pointer to the list of items.
6583  * @param[in] actions
6584  *   Pointer to the list of actions.
6585  * @param[out] error
6586  *   Pointer to the error structure.
6587  *
6588  * @return
6589  *   0 on success, a negative errno value otherwise and rte_errno is set.
6590  */
6591 static int
6592 __flow_dv_translate(struct rte_eth_dev *dev,
6593                     struct mlx5_flow *dev_flow,
6594                     const struct rte_flow_attr *attr,
6595                     const struct rte_flow_item items[],
6596                     const struct rte_flow_action actions[],
6597                     struct rte_flow_error *error)
6598 {
6599         struct mlx5_priv *priv = dev->data->dev_private;
6600         struct mlx5_dev_config *dev_conf = &priv->config;
6601         struct rte_flow *flow = dev_flow->flow;
6602         uint64_t item_flags = 0;
6603         uint64_t last_item = 0;
6604         uint64_t action_flags = 0;
6605         uint64_t priority = attr->priority;
6606         struct mlx5_flow_dv_matcher matcher = {
6607                 .mask = {
6608                         .size = sizeof(matcher.mask.buf),
6609                 },
6610         };
6611         int actions_n = 0;
6612         bool actions_end = false;
6613         struct mlx5_flow_dv_modify_hdr_resource mhdr_res = {
6614                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
6615                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
6616         };
6617         union flow_dv_attr flow_attr = { .attr = 0 };
6618         uint32_t tag_be;
6619         union mlx5_flow_tbl_key tbl_key;
6620         uint32_t modify_action_position = UINT32_MAX;
6621         void *match_mask = matcher.mask.buf;
6622         void *match_value = dev_flow->dv.value.buf;
6623         uint8_t next_protocol = 0xff;
6624         struct rte_vlan_hdr vlan = { 0 };
6625         uint32_t table;
6626         int ret = 0;
6627
6628         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
6629                                        &table, error);
6630         if (ret)
6631                 return ret;
6632         dev_flow->group = table;
6633         if (attr->transfer)
6634                 mhdr_res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
6635         if (priority == MLX5_FLOW_PRIO_RSVD)
6636                 priority = dev_conf->flow_prio - 1;
6637         for (; !actions_end ; actions++) {
6638                 const struct rte_flow_action_queue *queue;
6639                 const struct rte_flow_action_rss *rss;
6640                 const struct rte_flow_action *action = actions;
6641                 const struct rte_flow_action_count *count = action->conf;
6642                 const uint8_t *rss_key;
6643                 const struct rte_flow_action_jump *jump_data;
6644                 const struct rte_flow_action_meter *mtr;
6645                 struct mlx5_flow_tbl_resource *tbl;
6646                 uint32_t port_id = 0;
6647                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
6648                 int action_type = actions->type;
6649                 const struct rte_flow_action *found_action = NULL;
6650
6651                 switch (action_type) {
6652                 case RTE_FLOW_ACTION_TYPE_VOID:
6653                         break;
6654                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
6655                         if (flow_dv_translate_action_port_id(dev, action,
6656                                                              &port_id, error))
6657                                 return -rte_errno;
6658                         port_id_resource.port_id = port_id;
6659                         if (flow_dv_port_id_action_resource_register
6660                             (dev, &port_id_resource, dev_flow, error))
6661                                 return -rte_errno;
6662                         dev_flow->dv.actions[actions_n++] =
6663                                 dev_flow->dv.port_id_action->action;
6664                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
6665                         break;
6666                 case RTE_FLOW_ACTION_TYPE_FLAG:
6667                         action_flags |= MLX5_FLOW_ACTION_FLAG;
6668                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
6669                                 struct rte_flow_action_mark mark = {
6670                                         .id = MLX5_FLOW_MARK_DEFAULT,
6671                                 };
6672
6673                                 if (flow_dv_convert_action_mark(dev, &mark,
6674                                                                 &mhdr_res,
6675                                                                 error))
6676                                         return -rte_errno;
6677                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
6678                                 break;
6679                         }
6680                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
6681                         if (!dev_flow->dv.tag_resource)
6682                                 if (flow_dv_tag_resource_register
6683                                     (dev, tag_be, dev_flow, error))
6684                                         return -rte_errno;
6685                         dev_flow->dv.actions[actions_n++] =
6686                                 dev_flow->dv.tag_resource->action;
6687                         break;
6688                 case RTE_FLOW_ACTION_TYPE_MARK:
6689                         action_flags |= MLX5_FLOW_ACTION_MARK;
6690                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
6691                                 const struct rte_flow_action_mark *mark =
6692                                         (const struct rte_flow_action_mark *)
6693                                                 actions->conf;
6694
6695                                 if (flow_dv_convert_action_mark(dev, mark,
6696                                                                 &mhdr_res,
6697                                                                 error))
6698                                         return -rte_errno;
6699                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
6700                                 break;
6701                         }
6702                         /* Fall-through */
6703                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
6704                         /* Legacy (non-extensive) MARK action. */
6705                         tag_be = mlx5_flow_mark_set
6706                               (((const struct rte_flow_action_mark *)
6707                                (actions->conf))->id);
6708                         if (!dev_flow->dv.tag_resource)
6709                                 if (flow_dv_tag_resource_register
6710                                     (dev, tag_be, dev_flow, error))
6711                                         return -rte_errno;
6712                         dev_flow->dv.actions[actions_n++] =
6713                                 dev_flow->dv.tag_resource->action;
6714                         break;
6715                 case RTE_FLOW_ACTION_TYPE_SET_META:
6716                         if (flow_dv_convert_action_set_meta
6717                                 (dev, &mhdr_res, attr,
6718                                  (const struct rte_flow_action_set_meta *)
6719                                   actions->conf, error))
6720                                 return -rte_errno;
6721                         action_flags |= MLX5_FLOW_ACTION_SET_META;
6722                         break;
6723                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
6724                         if (flow_dv_convert_action_set_tag
6725                                 (dev, &mhdr_res,
6726                                  (const struct rte_flow_action_set_tag *)
6727                                   actions->conf, error))
6728                                 return -rte_errno;
6729                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
6730                         break;
6731                 case RTE_FLOW_ACTION_TYPE_DROP:
6732                         action_flags |= MLX5_FLOW_ACTION_DROP;
6733                         break;
6734                 case RTE_FLOW_ACTION_TYPE_QUEUE:
6735                         assert(flow->rss.queue);
6736                         queue = actions->conf;
6737                         flow->rss.queue_num = 1;
6738                         (*flow->rss.queue)[0] = queue->index;
6739                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
6740                         break;
6741                 case RTE_FLOW_ACTION_TYPE_RSS:
6742                         assert(flow->rss.queue);
6743                         rss = actions->conf;
6744                         if (flow->rss.queue)
6745                                 memcpy((*flow->rss.queue), rss->queue,
6746                                        rss->queue_num * sizeof(uint16_t));
6747                         flow->rss.queue_num = rss->queue_num;
6748                         /* NULL RSS key indicates default RSS key. */
6749                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
6750                         memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
6751                         /*
6752                          * rss->level and rss.types should be set in advance
6753                          * when expanding items for RSS.
6754                          */
6755                         action_flags |= MLX5_FLOW_ACTION_RSS;
6756                         break;
6757                 case RTE_FLOW_ACTION_TYPE_COUNT:
6758                         if (!dev_conf->devx) {
6759                                 rte_errno = ENOTSUP;
6760                                 goto cnt_err;
6761                         }
6762                         flow->counter = flow_dv_counter_alloc(dev,
6763                                                               count->shared,
6764                                                               count->id,
6765                                                               dev_flow->group);
6766                         if (flow->counter == NULL)
6767                                 goto cnt_err;
6768                         dev_flow->dv.actions[actions_n++] =
6769                                 flow->counter->action;
6770                         action_flags |= MLX5_FLOW_ACTION_COUNT;
6771                         break;
6772 cnt_err:
6773                         if (rte_errno == ENOTSUP)
6774                                 return rte_flow_error_set
6775                                               (error, ENOTSUP,
6776                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6777                                                NULL,
6778                                                "count action not supported");
6779                         else
6780                                 return rte_flow_error_set
6781                                                 (error, rte_errno,
6782                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6783                                                  action,
6784                                                  "cannot create counter"
6785                                                   " object.");
6786                         break;
6787                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
6788                         dev_flow->dv.actions[actions_n++] =
6789                                                 priv->sh->pop_vlan_action;
6790                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
6791                         break;
6792                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
6793                         flow_dev_get_vlan_info_from_items(items, &vlan);
6794                         vlan.eth_proto = rte_be_to_cpu_16
6795                              ((((const struct rte_flow_action_of_push_vlan *)
6796                                                    actions->conf)->ethertype));
6797                         found_action = mlx5_flow_find_action
6798                                         (actions + 1,
6799                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
6800                         if (found_action)
6801                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
6802                         found_action = mlx5_flow_find_action
6803                                         (actions + 1,
6804                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
6805                         if (found_action)
6806                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
6807                         if (flow_dv_create_action_push_vlan
6808                                             (dev, attr, &vlan, dev_flow, error))
6809                                 return -rte_errno;
6810                         dev_flow->dv.actions[actions_n++] =
6811                                            dev_flow->dv.push_vlan_res->action;
6812                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
6813                         break;
6814                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
6815                         /* of_vlan_push action handled this action */
6816                         assert(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN);
6817                         break;
6818                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
6819                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6820                                 break;
6821                         flow_dev_get_vlan_info_from_items(items, &vlan);
6822                         mlx5_update_vlan_vid_pcp(actions, &vlan);
6823                         /* If no VLAN push - this is a modify header action */
6824                         if (flow_dv_convert_action_modify_vlan_vid
6825                                                 (&mhdr_res, actions, error))
6826                                 return -rte_errno;
6827                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
6828                         break;
6829                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
6830                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
6831                         if (flow_dv_create_action_l2_encap(dev, actions,
6832                                                            dev_flow,
6833                                                            attr->transfer,
6834                                                            error))
6835                                 return -rte_errno;
6836                         dev_flow->dv.actions[actions_n++] =
6837                                 dev_flow->dv.encap_decap->verbs_action;
6838                         action_flags |= actions->type ==
6839                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
6840                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
6841                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
6842                         break;
6843                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
6844                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
6845                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
6846                                                            attr->transfer,
6847                                                            error))
6848                                 return -rte_errno;
6849                         dev_flow->dv.actions[actions_n++] =
6850                                 dev_flow->dv.encap_decap->verbs_action;
6851                         action_flags |= actions->type ==
6852                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
6853                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
6854                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
6855                         break;
6856                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
6857                         /* Handle encap with preceding decap. */
6858                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
6859                                 if (flow_dv_create_action_raw_encap
6860                                         (dev, actions, dev_flow, attr, error))
6861                                         return -rte_errno;
6862                                 dev_flow->dv.actions[actions_n++] =
6863                                         dev_flow->dv.encap_decap->verbs_action;
6864                         } else {
6865                                 /* Handle encap without preceding decap. */
6866                                 if (flow_dv_create_action_l2_encap
6867                                     (dev, actions, dev_flow, attr->transfer,
6868                                      error))
6869                                         return -rte_errno;
6870                                 dev_flow->dv.actions[actions_n++] =
6871                                         dev_flow->dv.encap_decap->verbs_action;
6872                         }
6873                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
6874                         break;
6875                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
6876                         /* Check if this decap is followed by encap. */
6877                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
6878                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
6879                                action++) {
6880                         }
6881                         /* Handle decap only if it isn't followed by encap. */
6882                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
6883                                 if (flow_dv_create_action_l2_decap
6884                                     (dev, dev_flow, attr->transfer, error))
6885                                         return -rte_errno;
6886                                 dev_flow->dv.actions[actions_n++] =
6887                                         dev_flow->dv.encap_decap->verbs_action;
6888                         }
6889                         /* If decap is followed by encap, handle it at encap. */
6890                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
6891                         break;
6892                 case RTE_FLOW_ACTION_TYPE_JUMP:
6893                         jump_data = action->conf;
6894                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
6895                                                        jump_data->group, &table,
6896                                                        error);
6897                         if (ret)
6898                                 return ret;
6899                         tbl = flow_dv_tbl_resource_get(dev, table,
6900                                                        attr->egress,
6901                                                        attr->transfer, error);
6902                         if (!tbl)
6903                                 return rte_flow_error_set
6904                                                 (error, errno,
6905                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6906                                                  NULL,
6907                                                  "cannot create jump action.");
6908                         if (flow_dv_jump_tbl_resource_register
6909                             (dev, tbl, dev_flow, error)) {
6910                                 flow_dv_tbl_resource_release(dev, tbl);
6911                                 return rte_flow_error_set
6912                                                 (error, errno,
6913                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6914                                                  NULL,
6915                                                  "cannot create jump action.");
6916                         }
6917                         dev_flow->dv.actions[actions_n++] =
6918                                 dev_flow->dv.jump->action;
6919                         action_flags |= MLX5_FLOW_ACTION_JUMP;
6920                         break;
6921                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
6922                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
6923                         if (flow_dv_convert_action_modify_mac
6924                                         (&mhdr_res, actions, error))
6925                                 return -rte_errno;
6926                         action_flags |= actions->type ==
6927                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
6928                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
6929                                         MLX5_FLOW_ACTION_SET_MAC_DST;
6930                         break;
6931                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
6932                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
6933                         if (flow_dv_convert_action_modify_ipv4
6934                                         (&mhdr_res, actions, error))
6935                                 return -rte_errno;
6936                         action_flags |= actions->type ==
6937                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
6938                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
6939                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
6940                         break;
6941                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
6942                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
6943                         if (flow_dv_convert_action_modify_ipv6
6944                                         (&mhdr_res, actions, error))
6945                                 return -rte_errno;
6946                         action_flags |= actions->type ==
6947                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
6948                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
6949                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
6950                         break;
6951                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
6952                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
6953                         if (flow_dv_convert_action_modify_tp
6954                                         (&mhdr_res, actions, items,
6955                                          &flow_attr, error))
6956                                 return -rte_errno;
6957                         action_flags |= actions->type ==
6958                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
6959                                         MLX5_FLOW_ACTION_SET_TP_SRC :
6960                                         MLX5_FLOW_ACTION_SET_TP_DST;
6961                         break;
6962                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
6963                         if (flow_dv_convert_action_modify_dec_ttl
6964                                         (&mhdr_res, items, &flow_attr, error))
6965                                 return -rte_errno;
6966                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
6967                         break;
6968                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
6969                         if (flow_dv_convert_action_modify_ttl
6970                                         (&mhdr_res, actions, items,
6971                                          &flow_attr, error))
6972                                 return -rte_errno;
6973                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
6974                         break;
6975                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
6976                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
6977                         if (flow_dv_convert_action_modify_tcp_seq
6978                                         (&mhdr_res, actions, error))
6979                                 return -rte_errno;
6980                         action_flags |= actions->type ==
6981                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
6982                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
6983                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
6984                         break;
6985
6986                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6987                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6988                         if (flow_dv_convert_action_modify_tcp_ack
6989                                         (&mhdr_res, actions, error))
6990                                 return -rte_errno;
6991                         action_flags |= actions->type ==
6992                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
6993                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
6994                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
6995                         break;
6996                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
6997                         if (flow_dv_convert_action_set_reg
6998                                         (&mhdr_res, actions, error))
6999                                 return -rte_errno;
7000                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7001                         break;
7002                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7003                         if (flow_dv_convert_action_copy_mreg
7004                                         (dev, &mhdr_res, actions, error))
7005                                 return -rte_errno;
7006                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7007                         break;
7008                 case RTE_FLOW_ACTION_TYPE_METER:
7009                         mtr = actions->conf;
7010                         if (!flow->meter) {
7011                                 flow->meter = mlx5_flow_meter_attach(priv,
7012                                                         mtr->mtr_id, attr,
7013                                                         error);
7014                                 if (!flow->meter)
7015                                         return rte_flow_error_set(error,
7016                                                 rte_errno,
7017                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7018                                                 NULL,
7019                                                 "meter not found "
7020                                                 "or invalid parameters");
7021                         }
7022                         /* Set the meter action. */
7023                         dev_flow->dv.actions[actions_n++] =
7024                                 flow->meter->mfts->meter_action;
7025                         action_flags |= MLX5_FLOW_ACTION_METER;
7026                         break;
7027                 case RTE_FLOW_ACTION_TYPE_END:
7028                         actions_end = true;
7029                         if (mhdr_res.actions_num) {
7030                                 /* create modify action if needed. */
7031                                 if (flow_dv_modify_hdr_resource_register
7032                                         (dev, &mhdr_res, dev_flow, error))
7033                                         return -rte_errno;
7034                                 dev_flow->dv.actions[modify_action_position] =
7035                                         dev_flow->dv.modify_hdr->verbs_action;
7036                         }
7037                         break;
7038                 default:
7039                         break;
7040                 }
7041                 if (mhdr_res.actions_num &&
7042                     modify_action_position == UINT32_MAX)
7043                         modify_action_position = actions_n++;
7044         }
7045         dev_flow->dv.actions_n = actions_n;
7046         dev_flow->actions = action_flags;
7047         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
7048                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
7049                 int item_type = items->type;
7050
7051                 switch (item_type) {
7052                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
7053                         flow_dv_translate_item_port_id(dev, match_mask,
7054                                                        match_value, items);
7055                         last_item = MLX5_FLOW_ITEM_PORT_ID;
7056                         break;
7057                 case RTE_FLOW_ITEM_TYPE_ETH:
7058                         flow_dv_translate_item_eth(match_mask, match_value,
7059                                                    items, tunnel);
7060                         matcher.priority = MLX5_PRIORITY_MAP_L2;
7061                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
7062                                              MLX5_FLOW_LAYER_OUTER_L2;
7063                         break;
7064                 case RTE_FLOW_ITEM_TYPE_VLAN:
7065                         flow_dv_translate_item_vlan(dev_flow,
7066                                                     match_mask, match_value,
7067                                                     items, tunnel);
7068                         matcher.priority = MLX5_PRIORITY_MAP_L2;
7069                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
7070                                               MLX5_FLOW_LAYER_INNER_VLAN) :
7071                                              (MLX5_FLOW_LAYER_OUTER_L2 |
7072                                               MLX5_FLOW_LAYER_OUTER_VLAN);
7073                         break;
7074                 case RTE_FLOW_ITEM_TYPE_IPV4:
7075                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7076                                                   &item_flags, &tunnel);
7077                         flow_dv_translate_item_ipv4(match_mask, match_value,
7078                                                     items, tunnel,
7079                                                     dev_flow->group);
7080                         matcher.priority = MLX5_PRIORITY_MAP_L3;
7081                         dev_flow->hash_fields |=
7082                                 mlx5_flow_hashfields_adjust
7083                                         (dev_flow, tunnel,
7084                                          MLX5_IPV4_LAYER_TYPES,
7085                                          MLX5_IPV4_IBV_RX_HASH);
7086                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
7087                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
7088                         if (items->mask != NULL &&
7089                             ((const struct rte_flow_item_ipv4 *)
7090                              items->mask)->hdr.next_proto_id) {
7091                                 next_protocol =
7092                                         ((const struct rte_flow_item_ipv4 *)
7093                                          (items->spec))->hdr.next_proto_id;
7094                                 next_protocol &=
7095                                         ((const struct rte_flow_item_ipv4 *)
7096                                          (items->mask))->hdr.next_proto_id;
7097                         } else {
7098                                 /* Reset for inner layer. */
7099                                 next_protocol = 0xff;
7100                         }
7101                         break;
7102                 case RTE_FLOW_ITEM_TYPE_IPV6:
7103                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7104                                                   &item_flags, &tunnel);
7105                         flow_dv_translate_item_ipv6(match_mask, match_value,
7106                                                     items, tunnel,
7107                                                     dev_flow->group);
7108                         matcher.priority = MLX5_PRIORITY_MAP_L3;
7109                         dev_flow->hash_fields |=
7110                                 mlx5_flow_hashfields_adjust
7111                                         (dev_flow, tunnel,
7112                                          MLX5_IPV6_LAYER_TYPES,
7113                                          MLX5_IPV6_IBV_RX_HASH);
7114                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
7115                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
7116                         if (items->mask != NULL &&
7117                             ((const struct rte_flow_item_ipv6 *)
7118                              items->mask)->hdr.proto) {
7119                                 next_protocol =
7120                                         ((const struct rte_flow_item_ipv6 *)
7121                                          items->spec)->hdr.proto;
7122                                 next_protocol &=
7123                                         ((const struct rte_flow_item_ipv6 *)
7124                                          items->mask)->hdr.proto;
7125                         } else {
7126                                 /* Reset for inner layer. */
7127                                 next_protocol = 0xff;
7128                         }
7129                         break;
7130                 case RTE_FLOW_ITEM_TYPE_TCP:
7131                         flow_dv_translate_item_tcp(match_mask, match_value,
7132                                                    items, tunnel);
7133                         matcher.priority = MLX5_PRIORITY_MAP_L4;
7134                         dev_flow->hash_fields |=
7135                                 mlx5_flow_hashfields_adjust
7136                                         (dev_flow, tunnel, ETH_RSS_TCP,
7137                                          IBV_RX_HASH_SRC_PORT_TCP |
7138                                          IBV_RX_HASH_DST_PORT_TCP);
7139                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
7140                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
7141                         break;
7142                 case RTE_FLOW_ITEM_TYPE_UDP:
7143                         flow_dv_translate_item_udp(match_mask, match_value,
7144                                                    items, tunnel);
7145                         matcher.priority = MLX5_PRIORITY_MAP_L4;
7146                         dev_flow->hash_fields |=
7147                                 mlx5_flow_hashfields_adjust
7148                                         (dev_flow, tunnel, ETH_RSS_UDP,
7149                                          IBV_RX_HASH_SRC_PORT_UDP |
7150                                          IBV_RX_HASH_DST_PORT_UDP);
7151                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
7152                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
7153                         break;
7154                 case RTE_FLOW_ITEM_TYPE_GRE:
7155                         flow_dv_translate_item_gre(match_mask, match_value,
7156                                                    items, tunnel);
7157                         last_item = MLX5_FLOW_LAYER_GRE;
7158                         break;
7159                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
7160                         flow_dv_translate_item_gre_key(match_mask,
7161                                                        match_value, items);
7162                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
7163                         break;
7164                 case RTE_FLOW_ITEM_TYPE_NVGRE:
7165                         flow_dv_translate_item_nvgre(match_mask, match_value,
7166                                                      items, tunnel);
7167                         last_item = MLX5_FLOW_LAYER_GRE;
7168                         break;
7169                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7170                         flow_dv_translate_item_vxlan(match_mask, match_value,
7171                                                      items, tunnel);
7172                         last_item = MLX5_FLOW_LAYER_VXLAN;
7173                         break;
7174                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7175                         flow_dv_translate_item_vxlan(match_mask, match_value,
7176                                                      items, tunnel);
7177                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7178                         break;
7179                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7180                         flow_dv_translate_item_geneve(match_mask, match_value,
7181                                                       items, tunnel);
7182                         last_item = MLX5_FLOW_LAYER_GENEVE;
7183                         break;
7184                 case RTE_FLOW_ITEM_TYPE_MPLS:
7185                         flow_dv_translate_item_mpls(match_mask, match_value,
7186                                                     items, last_item, tunnel);
7187                         last_item = MLX5_FLOW_LAYER_MPLS;
7188                         break;
7189                 case RTE_FLOW_ITEM_TYPE_MARK:
7190                         flow_dv_translate_item_mark(dev, match_mask,
7191                                                     match_value, items);
7192                         last_item = MLX5_FLOW_ITEM_MARK;
7193                         break;
7194                 case RTE_FLOW_ITEM_TYPE_META:
7195                         flow_dv_translate_item_meta(dev, match_mask,
7196                                                     match_value, attr, items);
7197                         last_item = MLX5_FLOW_ITEM_METADATA;
7198                         break;
7199                 case RTE_FLOW_ITEM_TYPE_ICMP:
7200                         flow_dv_translate_item_icmp(match_mask, match_value,
7201                                                     items, tunnel);
7202                         last_item = MLX5_FLOW_LAYER_ICMP;
7203                         break;
7204                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7205                         flow_dv_translate_item_icmp6(match_mask, match_value,
7206                                                       items, tunnel);
7207                         last_item = MLX5_FLOW_LAYER_ICMP6;
7208                         break;
7209                 case RTE_FLOW_ITEM_TYPE_TAG:
7210                         flow_dv_translate_item_tag(dev, match_mask,
7211                                                    match_value, items);
7212                         last_item = MLX5_FLOW_ITEM_TAG;
7213                         break;
7214                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7215                         flow_dv_translate_mlx5_item_tag(match_mask,
7216                                                         match_value, items);
7217                         last_item = MLX5_FLOW_ITEM_TAG;
7218                         break;
7219                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7220                         flow_dv_translate_item_tx_queue(dev, match_mask,
7221                                                         match_value,
7222                                                         items);
7223                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
7224                         break;
7225                 default:
7226                         break;
7227                 }
7228                 item_flags |= last_item;
7229         }
7230         /*
7231          * In case of ingress traffic when E-Switch mode is enabled,
7232          * we have two cases where we need to set the source port manually.
7233          * The first one, is in case of Nic steering rule, and the second is
7234          * E-Switch rule where no port_id item was found. In both cases
7235          * the source port is set according the current port in use.
7236          */
7237         if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
7238             (priv->representor || priv->master)) {
7239                 if (flow_dv_translate_item_port_id(dev, match_mask,
7240                                                    match_value, NULL))
7241                         return -rte_errno;
7242         }
7243         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
7244                                          dev_flow->dv.value.buf));
7245         dev_flow->layers = item_flags;
7246         /* Register matcher. */
7247         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
7248                                     matcher.mask.size);
7249         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
7250                                                      matcher.priority);
7251         /* reserved field no needs to be set to 0 here. */
7252         tbl_key.domain = attr->transfer;
7253         tbl_key.direction = attr->egress;
7254         tbl_key.table_id = dev_flow->group;
7255         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
7256                 return -rte_errno;
7257         return 0;
7258 }
7259
7260 /**
7261  * Apply the flow to the NIC, lock free,
7262  * (mutex should be acquired by caller).
7263  *
7264  * @param[in] dev
7265  *   Pointer to the Ethernet device structure.
7266  * @param[in, out] flow
7267  *   Pointer to flow structure.
7268  * @param[out] error
7269  *   Pointer to error structure.
7270  *
7271  * @return
7272  *   0 on success, a negative errno value otherwise and rte_errno is set.
7273  */
7274 static int
7275 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
7276                 struct rte_flow_error *error)
7277 {
7278         struct mlx5_flow_dv *dv;
7279         struct mlx5_flow *dev_flow;
7280         struct mlx5_priv *priv = dev->data->dev_private;
7281         int n;
7282         int err;
7283
7284         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
7285                 dv = &dev_flow->dv;
7286                 n = dv->actions_n;
7287                 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) {
7288                         if (dev_flow->transfer) {
7289                                 dv->actions[n++] = priv->sh->esw_drop_action;
7290                         } else {
7291                                 dv->hrxq = mlx5_hrxq_drop_new(dev);
7292                                 if (!dv->hrxq) {
7293                                         rte_flow_error_set
7294                                                 (error, errno,
7295                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7296                                                  NULL,
7297                                                  "cannot get drop hash queue");
7298                                         goto error;
7299                                 }
7300                                 dv->actions[n++] = dv->hrxq->action;
7301                         }
7302                 } else if (dev_flow->actions &
7303                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
7304                         struct mlx5_hrxq *hrxq;
7305
7306                         assert(flow->rss.queue);
7307                         hrxq = mlx5_hrxq_get(dev, flow->rss.key,
7308                                              MLX5_RSS_HASH_KEY_LEN,
7309                                              dev_flow->hash_fields,
7310                                              (*flow->rss.queue),
7311                                              flow->rss.queue_num);
7312                         if (!hrxq) {
7313                                 hrxq = mlx5_hrxq_new
7314                                         (dev, flow->rss.key,
7315                                          MLX5_RSS_HASH_KEY_LEN,
7316                                          dev_flow->hash_fields,
7317                                          (*flow->rss.queue),
7318                                          flow->rss.queue_num,
7319                                          !!(dev_flow->layers &
7320                                             MLX5_FLOW_LAYER_TUNNEL));
7321                         }
7322                         if (!hrxq) {
7323                                 rte_flow_error_set
7324                                         (error, rte_errno,
7325                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7326                                          "cannot get hash queue");
7327                                 goto error;
7328                         }
7329                         dv->hrxq = hrxq;
7330                         dv->actions[n++] = dv->hrxq->action;
7331                 }
7332                 dv->flow =
7333                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
7334                                                   (void *)&dv->value, n,
7335                                                   dv->actions);
7336                 if (!dv->flow) {
7337                         rte_flow_error_set(error, errno,
7338                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7339                                            NULL,
7340                                            "hardware refuses to create flow");
7341                         goto error;
7342                 }
7343                 if (priv->vmwa_context &&
7344                     dev_flow->dv.vf_vlan.tag &&
7345                     !dev_flow->dv.vf_vlan.created) {
7346                         /*
7347                          * The rule contains the VLAN pattern.
7348                          * For VF we are going to create VLAN
7349                          * interface to make hypervisor set correct
7350                          * e-Switch vport context.
7351                          */
7352                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
7353                 }
7354         }
7355         return 0;
7356 error:
7357         err = rte_errno; /* Save rte_errno before cleanup. */
7358         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
7359                 struct mlx5_flow_dv *dv = &dev_flow->dv;
7360                 if (dv->hrxq) {
7361                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
7362                                 mlx5_hrxq_drop_release(dev);
7363                         else
7364                                 mlx5_hrxq_release(dev, dv->hrxq);
7365                         dv->hrxq = NULL;
7366                 }
7367                 if (dev_flow->dv.vf_vlan.tag &&
7368                     dev_flow->dv.vf_vlan.created)
7369                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
7370         }
7371         rte_errno = err; /* Restore rte_errno. */
7372         return -rte_errno;
7373 }
7374
7375 /**
7376  * Release the flow matcher.
7377  *
7378  * @param dev
7379  *   Pointer to Ethernet device.
7380  * @param flow
7381  *   Pointer to mlx5_flow.
7382  *
7383  * @return
7384  *   1 while a reference on it exists, 0 when freed.
7385  */
7386 static int
7387 flow_dv_matcher_release(struct rte_eth_dev *dev,
7388                         struct mlx5_flow *flow)
7389 {
7390         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
7391
7392         assert(matcher->matcher_object);
7393         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
7394                 dev->data->port_id, (void *)matcher,
7395                 rte_atomic32_read(&matcher->refcnt));
7396         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
7397                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7398                            (matcher->matcher_object));
7399                 LIST_REMOVE(matcher, next);
7400                 /* table ref-- in release interface. */
7401                 flow_dv_tbl_resource_release(dev, matcher->tbl);
7402                 rte_free(matcher);
7403                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
7404                         dev->data->port_id, (void *)matcher);
7405                 return 0;
7406         }
7407         return 1;
7408 }
7409
7410 /**
7411  * Release an encap/decap resource.
7412  *
7413  * @param flow
7414  *   Pointer to mlx5_flow.
7415  *
7416  * @return
7417  *   1 while a reference on it exists, 0 when freed.
7418  */
7419 static int
7420 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
7421 {
7422         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
7423                                                 flow->dv.encap_decap;
7424
7425         assert(cache_resource->verbs_action);
7426         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
7427                 (void *)cache_resource,
7428                 rte_atomic32_read(&cache_resource->refcnt));
7429         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
7430                 claim_zero(mlx5_glue->destroy_flow_action
7431                                 (cache_resource->verbs_action));
7432                 LIST_REMOVE(cache_resource, next);
7433                 rte_free(cache_resource);
7434                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
7435                         (void *)cache_resource);
7436                 return 0;
7437         }
7438         return 1;
7439 }
7440
7441 /**
7442  * Release an jump to table action resource.
7443  *
7444  * @param dev
7445  *   Pointer to Ethernet device.
7446  * @param flow
7447  *   Pointer to mlx5_flow.
7448  *
7449  * @return
7450  *   1 while a reference on it exists, 0 when freed.
7451  */
7452 static int
7453 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
7454                                   struct mlx5_flow *flow)
7455 {
7456         struct mlx5_flow_dv_jump_tbl_resource *cache_resource = flow->dv.jump;
7457         struct mlx5_flow_tbl_data_entry *tbl_data =
7458                         container_of(cache_resource,
7459                                      struct mlx5_flow_tbl_data_entry, jump);
7460
7461         assert(cache_resource->action);
7462         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
7463                 (void *)cache_resource,
7464                 rte_atomic32_read(&cache_resource->refcnt));
7465         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
7466                 claim_zero(mlx5_glue->destroy_flow_action
7467                                 (cache_resource->action));
7468                 /* jump action memory free is inside the table release. */
7469                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
7470                 DRV_LOG(DEBUG, "jump table resource %p: removed",
7471                         (void *)cache_resource);
7472                 return 0;
7473         }
7474         return 1;
7475 }
7476
7477 /**
7478  * Release a modify-header resource.
7479  *
7480  * @param flow
7481  *   Pointer to mlx5_flow.
7482  *
7483  * @return
7484  *   1 while a reference on it exists, 0 when freed.
7485  */
7486 static int
7487 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
7488 {
7489         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
7490                                                 flow->dv.modify_hdr;
7491
7492         assert(cache_resource->verbs_action);
7493         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
7494                 (void *)cache_resource,
7495                 rte_atomic32_read(&cache_resource->refcnt));
7496         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
7497                 claim_zero(mlx5_glue->destroy_flow_action
7498                                 (cache_resource->verbs_action));
7499                 LIST_REMOVE(cache_resource, next);
7500                 rte_free(cache_resource);
7501                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
7502                         (void *)cache_resource);
7503                 return 0;
7504         }
7505         return 1;
7506 }
7507
7508 /**
7509  * Release port ID action resource.
7510  *
7511  * @param flow
7512  *   Pointer to mlx5_flow.
7513  *
7514  * @return
7515  *   1 while a reference on it exists, 0 when freed.
7516  */
7517 static int
7518 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
7519 {
7520         struct mlx5_flow_dv_port_id_action_resource *cache_resource =
7521                 flow->dv.port_id_action;
7522
7523         assert(cache_resource->action);
7524         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
7525                 (void *)cache_resource,
7526                 rte_atomic32_read(&cache_resource->refcnt));
7527         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
7528                 claim_zero(mlx5_glue->destroy_flow_action
7529                                 (cache_resource->action));
7530                 LIST_REMOVE(cache_resource, next);
7531                 rte_free(cache_resource);
7532                 DRV_LOG(DEBUG, "port id action resource %p: removed",
7533                         (void *)cache_resource);
7534                 return 0;
7535         }
7536         return 1;
7537 }
7538
7539 /**
7540  * Release push vlan action resource.
7541  *
7542  * @param flow
7543  *   Pointer to mlx5_flow.
7544  *
7545  * @return
7546  *   1 while a reference on it exists, 0 when freed.
7547  */
7548 static int
7549 flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow)
7550 {
7551         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource =
7552                 flow->dv.push_vlan_res;
7553
7554         assert(cache_resource->action);
7555         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
7556                 (void *)cache_resource,
7557                 rte_atomic32_read(&cache_resource->refcnt));
7558         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
7559                 claim_zero(mlx5_glue->destroy_flow_action
7560                                 (cache_resource->action));
7561                 LIST_REMOVE(cache_resource, next);
7562                 rte_free(cache_resource);
7563                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
7564                         (void *)cache_resource);
7565                 return 0;
7566         }
7567         return 1;
7568 }
7569
7570 /**
7571  * Remove the flow from the NIC but keeps it in memory.
7572  * Lock free, (mutex should be acquired by caller).
7573  *
7574  * @param[in] dev
7575  *   Pointer to Ethernet device.
7576  * @param[in, out] flow
7577  *   Pointer to flow structure.
7578  */
7579 static void
7580 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
7581 {
7582         struct mlx5_flow_dv *dv;
7583         struct mlx5_flow *dev_flow;
7584
7585         if (!flow)
7586                 return;
7587         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
7588                 dv = &dev_flow->dv;
7589                 if (dv->flow) {
7590                         claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
7591                         dv->flow = NULL;
7592                 }
7593                 if (dv->hrxq) {
7594                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
7595                                 mlx5_hrxq_drop_release(dev);
7596                         else
7597                                 mlx5_hrxq_release(dev, dv->hrxq);
7598                         dv->hrxq = NULL;
7599                 }
7600                 if (dev_flow->dv.vf_vlan.tag &&
7601                     dev_flow->dv.vf_vlan.created)
7602                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
7603         }
7604 }
7605
7606 /**
7607  * Remove the flow from the NIC and the memory.
7608  * Lock free, (mutex should be acquired by caller).
7609  *
7610  * @param[in] dev
7611  *   Pointer to the Ethernet device structure.
7612  * @param[in, out] flow
7613  *   Pointer to flow structure.
7614  */
7615 static void
7616 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
7617 {
7618         struct mlx5_flow *dev_flow;
7619
7620         if (!flow)
7621                 return;
7622         __flow_dv_remove(dev, flow);
7623         if (flow->counter) {
7624                 flow_dv_counter_release(dev, flow->counter);
7625                 flow->counter = NULL;
7626         }
7627         if (flow->meter) {
7628                 mlx5_flow_meter_detach(flow->meter);
7629                 flow->meter = NULL;
7630         }
7631         while (!LIST_EMPTY(&flow->dev_flows)) {
7632                 dev_flow = LIST_FIRST(&flow->dev_flows);
7633                 LIST_REMOVE(dev_flow, next);
7634                 if (dev_flow->dv.matcher)
7635                         flow_dv_matcher_release(dev, dev_flow);
7636                 if (dev_flow->dv.encap_decap)
7637                         flow_dv_encap_decap_resource_release(dev_flow);
7638                 if (dev_flow->dv.modify_hdr)
7639                         flow_dv_modify_hdr_resource_release(dev_flow);
7640                 if (dev_flow->dv.jump)
7641                         flow_dv_jump_tbl_resource_release(dev, dev_flow);
7642                 if (dev_flow->dv.port_id_action)
7643                         flow_dv_port_id_action_resource_release(dev_flow);
7644                 if (dev_flow->dv.push_vlan_res)
7645                         flow_dv_push_vlan_action_resource_release(dev_flow);
7646                 if (dev_flow->dv.tag_resource)
7647                         flow_dv_tag_release(dev, dev_flow->dv.tag_resource);
7648                 rte_free(dev_flow);
7649         }
7650 }
7651
7652 /**
7653  * Query a dv flow  rule for its statistics via devx.
7654  *
7655  * @param[in] dev
7656  *   Pointer to Ethernet device.
7657  * @param[in] flow
7658  *   Pointer to the sub flow.
7659  * @param[out] data
7660  *   data retrieved by the query.
7661  * @param[out] error
7662  *   Perform verbose error reporting if not NULL.
7663  *
7664  * @return
7665  *   0 on success, a negative errno value otherwise and rte_errno is set.
7666  */
7667 static int
7668 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
7669                     void *data, struct rte_flow_error *error)
7670 {
7671         struct mlx5_priv *priv = dev->data->dev_private;
7672         struct rte_flow_query_count *qc = data;
7673
7674         if (!priv->config.devx)
7675                 return rte_flow_error_set(error, ENOTSUP,
7676                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7677                                           NULL,
7678                                           "counters are not supported");
7679         if (flow->counter) {
7680                 uint64_t pkts, bytes;
7681                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
7682                                                &bytes);
7683
7684                 if (err)
7685                         return rte_flow_error_set(error, -err,
7686                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7687                                         NULL, "cannot read counters");
7688                 qc->hits_set = 1;
7689                 qc->bytes_set = 1;
7690                 qc->hits = pkts - flow->counter->hits;
7691                 qc->bytes = bytes - flow->counter->bytes;
7692                 if (qc->reset) {
7693                         flow->counter->hits = pkts;
7694                         flow->counter->bytes = bytes;
7695                 }
7696                 return 0;
7697         }
7698         return rte_flow_error_set(error, EINVAL,
7699                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7700                                   NULL,
7701                                   "counters are not available");
7702 }
7703
7704 /**
7705  * Query a flow.
7706  *
7707  * @see rte_flow_query()
7708  * @see rte_flow_ops
7709  */
7710 static int
7711 flow_dv_query(struct rte_eth_dev *dev,
7712               struct rte_flow *flow __rte_unused,
7713               const struct rte_flow_action *actions __rte_unused,
7714               void *data __rte_unused,
7715               struct rte_flow_error *error __rte_unused)
7716 {
7717         int ret = -EINVAL;
7718
7719         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7720                 switch (actions->type) {
7721                 case RTE_FLOW_ACTION_TYPE_VOID:
7722                         break;
7723                 case RTE_FLOW_ACTION_TYPE_COUNT:
7724                         ret = flow_dv_query_count(dev, flow, data, error);
7725                         break;
7726                 default:
7727                         return rte_flow_error_set(error, ENOTSUP,
7728                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7729                                                   actions,
7730                                                   "action not supported");
7731                 }
7732         }
7733         return ret;
7734 }
7735
7736 /**
7737  * Destroy the meter table set.
7738  * Lock free, (mutex should be acquired by caller).
7739  *
7740  * @param[in] dev
7741  *   Pointer to Ethernet device.
7742  * @param[in] tbl
7743  *   Pointer to the meter table set.
7744  *
7745  * @return
7746  *   Always 0.
7747  */
7748 static int
7749 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
7750                         struct mlx5_meter_domains_infos *tbl)
7751 {
7752         struct mlx5_priv *priv = dev->data->dev_private;
7753         struct mlx5_meter_domains_infos *mtd =
7754                                 (struct mlx5_meter_domains_infos *)tbl;
7755
7756         if (!mtd || !priv->config.dv_flow_en)
7757                 return 0;
7758         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
7759                 claim_zero(mlx5_glue->dv_destroy_flow
7760                           (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
7761         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
7762                 claim_zero(mlx5_glue->dv_destroy_flow
7763                           (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
7764         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
7765                 claim_zero(mlx5_glue->dv_destroy_flow
7766                           (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
7767         if (mtd->egress.color_matcher)
7768                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7769                           (mtd->egress.color_matcher));
7770         if (mtd->egress.any_matcher)
7771                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7772                           (mtd->egress.any_matcher));
7773         if (mtd->egress.tbl)
7774                 claim_zero(flow_dv_tbl_resource_release(dev,
7775                                                         mtd->egress.tbl));
7776         if (mtd->ingress.color_matcher)
7777                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7778                           (mtd->ingress.color_matcher));
7779         if (mtd->ingress.any_matcher)
7780                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7781                           (mtd->ingress.any_matcher));
7782         if (mtd->ingress.tbl)
7783                 claim_zero(flow_dv_tbl_resource_release(dev,
7784                                                         mtd->ingress.tbl));
7785         if (mtd->transfer.color_matcher)
7786                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7787                           (mtd->transfer.color_matcher));
7788         if (mtd->transfer.any_matcher)
7789                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
7790                           (mtd->transfer.any_matcher));
7791         if (mtd->transfer.tbl)
7792                 claim_zero(flow_dv_tbl_resource_release(dev,
7793                                                         mtd->transfer.tbl));
7794         if (mtd->drop_actn)
7795                 claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn));
7796         rte_free(mtd);
7797         return 0;
7798 }
7799
7800 /* Number of meter flow actions, count and jump or count and drop. */
7801 #define METER_ACTIONS 2
7802
7803 /**
7804  * Create specify domain meter table and suffix table.
7805  *
7806  * @param[in] dev
7807  *   Pointer to Ethernet device.
7808  * @param[in,out] mtb
7809  *   Pointer to DV meter table set.
7810  * @param[in] egress
7811  *   Table attribute.
7812  * @param[in] transfer
7813  *   Table attribute.
7814  * @param[in] color_reg_c_idx
7815  *   Reg C index for color match.
7816  *
7817  * @return
7818  *   0 on success, -1 otherwise and rte_errno is set.
7819  */
7820 static int
7821 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
7822                            struct mlx5_meter_domains_infos *mtb,
7823                            uint8_t egress, uint8_t transfer,
7824                            uint32_t color_reg_c_idx)
7825 {
7826         struct mlx5_priv *priv = dev->data->dev_private;
7827         struct mlx5_ibv_shared *sh = priv->sh;
7828         struct mlx5_flow_dv_match_params mask = {
7829                 .size = sizeof(mask.buf),
7830         };
7831         struct mlx5_flow_dv_match_params value = {
7832                 .size = sizeof(value.buf),
7833         };
7834         struct mlx5dv_flow_matcher_attr dv_attr = {
7835                 .type = IBV_FLOW_ATTR_NORMAL,
7836                 .priority = 0,
7837                 .match_criteria_enable = 0,
7838                 .match_mask = (void *)&mask,
7839         };
7840         void *actions[METER_ACTIONS];
7841         struct mlx5_flow_tbl_resource **sfx_tbl;
7842         struct mlx5_meter_domain_info *dtb;
7843         struct rte_flow_error error;
7844         int i = 0;
7845
7846         if (transfer) {
7847                 sfx_tbl = &sh->fdb_mtr_sfx_tbl;
7848                 dtb = &mtb->transfer;
7849         } else if (egress) {
7850                 sfx_tbl = &sh->tx_mtr_sfx_tbl;
7851                 dtb = &mtb->egress;
7852         } else {
7853                 sfx_tbl = &sh->rx_mtr_sfx_tbl;
7854                 dtb = &mtb->ingress;
7855         }
7856         /* If the suffix table in missing, create it. */
7857         if (!(*sfx_tbl)) {
7858                 *sfx_tbl = flow_dv_tbl_resource_get(dev,
7859                                                 MLX5_FLOW_TABLE_LEVEL_SUFFIX,
7860                                                 egress, transfer, &error);
7861                 if (!(*sfx_tbl)) {
7862                         DRV_LOG(ERR, "Failed to create meter suffix table.");
7863                         return -1;
7864                 }
7865         }
7866         /* Create the meter table with METER level. */
7867         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
7868                                             egress, transfer, &error);
7869         if (!dtb->tbl) {
7870                 DRV_LOG(ERR, "Failed to create meter policer table.");
7871                 return -1;
7872         }
7873         /* Create matchers, Any and Color. */
7874         dv_attr.priority = 3;
7875         dv_attr.match_criteria_enable = 0;
7876         dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
7877                                                              &dv_attr,
7878                                                              dtb->tbl->obj);
7879         if (!dtb->any_matcher) {
7880                 DRV_LOG(ERR, "Failed to create meter"
7881                              " policer default matcher.");
7882                 goto error_exit;
7883         }
7884         dv_attr.priority = 0;
7885         dv_attr.match_criteria_enable =
7886                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7887         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
7888                                rte_col_2_mlx5_col(RTE_COLORS), UINT32_MAX);
7889         dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
7890                                                                &dv_attr,
7891                                                                dtb->tbl->obj);
7892         if (!dtb->color_matcher) {
7893                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
7894                 goto error_exit;
7895         }
7896         if (mtb->count_actns[RTE_MTR_DROPPED])
7897                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
7898         actions[i++] = mtb->drop_actn;
7899         /* Default rule: lowest priority, match any, actions: drop. */
7900         dtb->policer_rules[RTE_MTR_DROPPED] =
7901                         mlx5_glue->dv_create_flow(dtb->any_matcher,
7902                                                  (void *)&value, i, actions);
7903         if (!dtb->policer_rules[RTE_MTR_DROPPED]) {
7904                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
7905                 goto error_exit;
7906         }
7907         return 0;
7908 error_exit:
7909         return -1;
7910 }
7911
7912 /**
7913  * Create the needed meter and suffix tables.
7914  * Lock free, (mutex should be acquired by caller).
7915  *
7916  * @param[in] dev
7917  *   Pointer to Ethernet device.
7918  * @param[in] fm
7919  *   Pointer to the flow meter.
7920  *
7921  * @return
7922  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
7923  */
7924 static struct mlx5_meter_domains_infos *
7925 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
7926                        const struct mlx5_flow_meter *fm)
7927 {
7928         struct mlx5_priv *priv = dev->data->dev_private;
7929         struct mlx5_meter_domains_infos *mtb;
7930         int ret;
7931         int i;
7932
7933         if (!priv->mtr_en) {
7934                 rte_errno = ENOTSUP;
7935                 return NULL;
7936         }
7937         mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0);
7938         if (!mtb) {
7939                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
7940                 return NULL;
7941         }
7942         /* Create meter count actions */
7943         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
7944                 if (!fm->policer_stats.cnt[i])
7945                         continue;
7946                 mtb->count_actns[i] = fm->policer_stats.cnt[i]->action;
7947         }
7948         /* Create drop action. */
7949         mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop();
7950         if (!mtb->drop_actn) {
7951                 DRV_LOG(ERR, "Failed to create drop action.");
7952                 goto error_exit;
7953         }
7954         /* Egress meter table. */
7955         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
7956         if (ret) {
7957                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
7958                 goto error_exit;
7959         }
7960         /* Ingress meter table. */
7961         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
7962         if (ret) {
7963                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
7964                 goto error_exit;
7965         }
7966         /* FDB meter table. */
7967         if (priv->config.dv_esw_en) {
7968                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
7969                                                  priv->mtr_color_reg);
7970                 if (ret) {
7971                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
7972                         goto error_exit;
7973                 }
7974         }
7975         return mtb;
7976 error_exit:
7977         flow_dv_destroy_mtr_tbl(dev, mtb);
7978         return NULL;
7979 }
7980
7981 /**
7982  * Destroy domain policer rule.
7983  *
7984  * @param[in] dt
7985  *   Pointer to domain table.
7986  */
7987 static void
7988 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
7989 {
7990         int i;
7991
7992         for (i = 0; i < RTE_MTR_DROPPED; i++) {
7993                 if (dt->policer_rules[i]) {
7994                         claim_zero(mlx5_glue->dv_destroy_flow
7995                                   (dt->policer_rules[i]));
7996                         dt->policer_rules[i] = NULL;
7997                 }
7998         }
7999         if (dt->jump_actn) {
8000                 claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn));
8001                 dt->jump_actn = NULL;
8002         }
8003 }
8004
8005 /**
8006  * Destroy policer rules.
8007  *
8008  * @param[in] dev
8009  *   Pointer to Ethernet device.
8010  * @param[in] fm
8011  *   Pointer to flow meter structure.
8012  * @param[in] attr
8013  *   Pointer to flow attributes.
8014  *
8015  * @return
8016  *   Always 0.
8017  */
8018 static int
8019 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
8020                               const struct mlx5_flow_meter *fm,
8021                               const struct rte_flow_attr *attr)
8022 {
8023         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
8024
8025         if (!mtb)
8026                 return 0;
8027         if (attr->egress)
8028                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
8029         if (attr->ingress)
8030                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
8031         if (attr->transfer)
8032                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
8033         return 0;
8034 }
8035
8036 /**
8037  * Create specify domain meter policer rule.
8038  *
8039  * @param[in] fm
8040  *   Pointer to flow meter structure.
8041  * @param[in] mtb
8042  *   Pointer to DV meter table set.
8043  * @param[in] sfx_tb
8044  *   Pointer to suffix table.
8045  * @param[in] mtr_reg_c
8046  *   Color match REG_C.
8047  *
8048  * @return
8049  *   0 on success, -1 otherwise.
8050  */
8051 static int
8052 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
8053                                     struct mlx5_meter_domain_info *dtb,
8054                                     struct mlx5_flow_tbl_resource *sfx_tb,
8055                                     uint8_t mtr_reg_c)
8056 {
8057         struct mlx5_flow_dv_match_params matcher = {
8058                 .size = sizeof(matcher.buf),
8059         };
8060         struct mlx5_flow_dv_match_params value = {
8061                 .size = sizeof(value.buf),
8062         };
8063         struct mlx5_meter_domains_infos *mtb = fm->mfts;
8064         void *actions[METER_ACTIONS];
8065         int i;
8066
8067         /* Create jump action. */
8068         if (!sfx_tb)
8069                 return -1;
8070         if (!dtb->jump_actn)
8071                 dtb->jump_actn =
8072                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
8073                                                         (sfx_tb->obj);
8074         if (!dtb->jump_actn) {
8075                 DRV_LOG(ERR, "Failed to create policer jump action.");
8076                 goto error;
8077         }
8078         for (i = 0; i < RTE_MTR_DROPPED; i++) {
8079                 int j = 0;
8080
8081                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
8082                                        rte_col_2_mlx5_col(i), UINT32_MAX);
8083                 if (mtb->count_actns[i])
8084                         actions[j++] = mtb->count_actns[i];
8085                 if (fm->params.action[i] == MTR_POLICER_ACTION_DROP)
8086                         actions[j++] = mtb->drop_actn;
8087                 else
8088                         actions[j++] = dtb->jump_actn;
8089                 dtb->policer_rules[i] =
8090                         mlx5_glue->dv_create_flow(dtb->color_matcher,
8091                                                  (void *)&value,
8092                                                   j, actions);
8093                 if (!dtb->policer_rules[i]) {
8094                         DRV_LOG(ERR, "Failed to create policer rule.");
8095                         goto error;
8096                 }
8097         }
8098         return 0;
8099 error:
8100         rte_errno = errno;
8101         return -1;
8102 }
8103
8104 /**
8105  * Create policer rules.
8106  *
8107  * @param[in] dev
8108  *   Pointer to Ethernet device.
8109  * @param[in] fm
8110  *   Pointer to flow meter structure.
8111  * @param[in] attr
8112  *   Pointer to flow attributes.
8113  *
8114  * @return
8115  *   0 on success, -1 otherwise.
8116  */
8117 static int
8118 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
8119                              struct mlx5_flow_meter *fm,
8120                              const struct rte_flow_attr *attr)
8121 {
8122         struct mlx5_priv *priv = dev->data->dev_private;
8123         struct mlx5_meter_domains_infos *mtb = fm->mfts;
8124         int ret;
8125
8126         if (attr->egress) {
8127                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
8128                                                 priv->sh->tx_mtr_sfx_tbl,
8129                                                 priv->mtr_color_reg);
8130                 if (ret) {
8131                         DRV_LOG(ERR, "Failed to create egress policer.");
8132                         goto error;
8133                 }
8134         }
8135         if (attr->ingress) {
8136                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
8137                                                 priv->sh->rx_mtr_sfx_tbl,
8138                                                 priv->mtr_color_reg);
8139                 if (ret) {
8140                         DRV_LOG(ERR, "Failed to create ingress policer.");
8141                         goto error;
8142                 }
8143         }
8144         if (attr->transfer) {
8145                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
8146                                                 priv->sh->fdb_mtr_sfx_tbl,
8147                                                 priv->mtr_color_reg);
8148                 if (ret) {
8149                         DRV_LOG(ERR, "Failed to create transfer policer.");
8150                         goto error;
8151                 }
8152         }
8153         return 0;
8154 error:
8155         flow_dv_destroy_policer_rules(dev, fm, attr);
8156         return -1;
8157 }
8158
8159 /**
8160  * Query a devx counter.
8161  *
8162  * @param[in] dev
8163  *   Pointer to the Ethernet device structure.
8164  * @param[in] cnt
8165  *   Pointer to the flow counter.
8166  * @param[in] clear
8167  *   Set to clear the counter statistics.
8168  * @param[out] pkts
8169  *   The statistics value of packets.
8170  * @param[out] bytes
8171  *   The statistics value of bytes.
8172  *
8173  * @return
8174  *   0 on success, otherwise return -1.
8175  */
8176 static int
8177 flow_dv_counter_query(struct rte_eth_dev *dev,
8178                       struct mlx5_flow_counter *cnt, bool clear,
8179                       uint64_t *pkts, uint64_t *bytes)
8180 {
8181         struct mlx5_priv *priv = dev->data->dev_private;
8182         uint64_t inn_pkts, inn_bytes;
8183         int ret;
8184
8185         if (!priv->config.devx)
8186                 return -1;
8187         ret = _flow_dv_query_count(dev, cnt, &inn_pkts, &inn_bytes);
8188         if (ret)
8189                 return -1;
8190         *pkts = inn_pkts - cnt->hits;
8191         *bytes = inn_bytes - cnt->bytes;
8192         if (clear) {
8193                 cnt->hits = inn_pkts;
8194                 cnt->bytes = inn_bytes;
8195         }
8196         return 0;
8197 }
8198
8199 /*
8200  * Mutex-protected thunk to lock-free  __flow_dv_translate().
8201  */
8202 static int
8203 flow_dv_translate(struct rte_eth_dev *dev,
8204                   struct mlx5_flow *dev_flow,
8205                   const struct rte_flow_attr *attr,
8206                   const struct rte_flow_item items[],
8207                   const struct rte_flow_action actions[],
8208                   struct rte_flow_error *error)
8209 {
8210         int ret;
8211
8212         flow_dv_shared_lock(dev);
8213         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
8214         flow_dv_shared_unlock(dev);
8215         return ret;
8216 }
8217
8218 /*
8219  * Mutex-protected thunk to lock-free  __flow_dv_apply().
8220  */
8221 static int
8222 flow_dv_apply(struct rte_eth_dev *dev,
8223               struct rte_flow *flow,
8224               struct rte_flow_error *error)
8225 {
8226         int ret;
8227
8228         flow_dv_shared_lock(dev);
8229         ret = __flow_dv_apply(dev, flow, error);
8230         flow_dv_shared_unlock(dev);
8231         return ret;
8232 }
8233
8234 /*
8235  * Mutex-protected thunk to lock-free __flow_dv_remove().
8236  */
8237 static void
8238 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
8239 {
8240         flow_dv_shared_lock(dev);
8241         __flow_dv_remove(dev, flow);
8242         flow_dv_shared_unlock(dev);
8243 }
8244
8245 /*
8246  * Mutex-protected thunk to lock-free __flow_dv_destroy().
8247  */
8248 static void
8249 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
8250 {
8251         flow_dv_shared_lock(dev);
8252         __flow_dv_destroy(dev, flow);
8253         flow_dv_shared_unlock(dev);
8254 }
8255
8256 /*
8257  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
8258  */
8259 static struct mlx5_flow_counter *
8260 flow_dv_counter_allocate(struct rte_eth_dev *dev)
8261 {
8262         struct mlx5_flow_counter *cnt;
8263
8264         flow_dv_shared_lock(dev);
8265         cnt = flow_dv_counter_alloc(dev, 0, 0, 1);
8266         flow_dv_shared_unlock(dev);
8267         return cnt;
8268 }
8269
8270 /*
8271  * Mutex-protected thunk to lock-free flow_dv_counter_release().
8272  */
8273 static void
8274 flow_dv_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt)
8275 {
8276         flow_dv_shared_lock(dev);
8277         flow_dv_counter_release(dev, cnt);
8278         flow_dv_shared_unlock(dev);
8279 }
8280
8281 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
8282         .validate = flow_dv_validate,
8283         .prepare = flow_dv_prepare,
8284         .translate = flow_dv_translate,
8285         .apply = flow_dv_apply,
8286         .remove = flow_dv_remove,
8287         .destroy = flow_dv_destroy,
8288         .query = flow_dv_query,
8289         .create_mtr_tbls = flow_dv_create_mtr_tbl,
8290         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
8291         .create_policer_rules = flow_dv_create_policer_rules,
8292         .destroy_policer_rules = flow_dv_destroy_policer_rules,
8293         .counter_alloc = flow_dv_counter_allocate,
8294         .counter_free = flow_dv_counter_free,
8295         .counter_query = flow_dv_counter_query,
8296 };
8297
8298 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */