drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29 #include <rte_vxlan.h>
  30
  31 #include "mlx5.h"
  32 #include "mlx5_defs.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35 #include "mlx5_prm.h"
  36 #include "mlx5_rxtx.h"
  37
  38 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  39
  40 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  41 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  42 #endif
  43
  44 #ifndef HAVE_MLX5DV_DR_ESWITCH
  45 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  46 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  47 #endif
  48 #endif
  49
  50 #ifndef HAVE_MLX5DV_DR
  51 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  52 #endif
  53
  54 /* VLAN header definitions */
  55 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  56 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  57 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  58 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  59 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  60
  61 union flow_dv_attr {
  62         struct {
  63                 uint32_t valid:1;
  64                 uint32_t ipv4:1;
  65                 uint32_t ipv6:1;
  66                 uint32_t tcp:1;
  67                 uint32_t udp:1;
  68                 uint32_t reserved:27;
  69         };
  70         uint32_t attr;
  71 };
  72
  73 /**
  74  * Initialize flow attributes structure according to flow items' types.
  75  *
  76  * @param[in] item
  77  *   Pointer to item specification.
  78  * @param[out] attr
  79  *   Pointer to flow attributes structure.
  80  */
  81 static void
  82 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  83 {
  84         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  85                 switch (item->type) {
  86                 case RTE_FLOW_ITEM_TYPE_IPV4:
  87                         attr->ipv4 = 1;
  88                         break;
  89                 case RTE_FLOW_ITEM_TYPE_IPV6:
  90                         attr->ipv6 = 1;
  91                         break;
  92                 case RTE_FLOW_ITEM_TYPE_UDP:
  93                         attr->udp = 1;
  94                         break;
  95                 case RTE_FLOW_ITEM_TYPE_TCP:
  96                         attr->tcp = 1;
  97                         break;
  98                 default:
  99                         break;
 100                 }
 101         }
 102         attr->valid = 1;
 103 }
 104
 105 struct field_modify_info {
 106         uint32_t size; /* Size of field in protocol header, in bytes. */
 107         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 108         enum mlx5_modification_field id;
 109 };
 110
 111 struct field_modify_info modify_eth[] = {
 112         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 113         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 114         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 115         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 116         {0, 0, 0},
 117 };
 118
 119 struct field_modify_info modify_vlan_out_first_vid[] = {
 120         /* Size in bits !!! */
 121         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 122         {0, 0, 0},
 123 };
 124
 125 struct field_modify_info modify_ipv4[] = {
 126         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 127         {4, 12, MLX5_MODI_OUT_SIPV4},
 128         {4, 16, MLX5_MODI_OUT_DIPV4},
 129         {0, 0, 0},
 130 };
 131
 132 struct field_modify_info modify_ipv6[] = {
 133         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 134         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 135         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 136         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 137         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 138         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 139         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 140         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 141         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 142         {0, 0, 0},
 143 };
 144
 145 struct field_modify_info modify_udp[] = {
 146         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 147         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 148         {0, 0, 0},
 149 };
 150
 151 struct field_modify_info modify_tcp[] = {
 152         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 153         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 154         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 155         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 156         {0, 0, 0},
 157 };
 158
 159 static void
 160 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 161                           uint8_t next_protocol, uint64_t *item_flags,
 162                           int *tunnel)
 163 {
 164         assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 165                item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 166         if (next_protocol == IPPROTO_IPIP) {
 167                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 168                 *tunnel = 1;
 169         }
 170         if (next_protocol == IPPROTO_IPV6) {
 171                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 172                 *tunnel = 1;
 173         }
 174 }
 175
 176 /**
 177  * Acquire the synchronizing object to protect multithreaded access
 178  * to shared dv context. Lock occurs only if context is actually
 179  * shared, i.e. we have multiport IB device and representors are
 180  * created.
 181  *
 182  * @param[in] dev
 183  *   Pointer to the rte_eth_dev structure.
 184  */
 185 static void
 186 flow_dv_shared_lock(struct rte_eth_dev *dev)
 187 {
 188         struct mlx5_priv *priv = dev->data->dev_private;
 189         struct mlx5_ibv_shared *sh = priv->sh;
 190
 191         if (sh->dv_refcnt > 1) {
 192                 int ret;
 193
 194                 ret = pthread_mutex_lock(&sh->dv_mutex);
 195                 assert(!ret);
 196                 (void)ret;
 197         }
 198 }
 199
 200 static void
 201 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 202 {
 203         struct mlx5_priv *priv = dev->data->dev_private;
 204         struct mlx5_ibv_shared *sh = priv->sh;
 205
 206         if (sh->dv_refcnt > 1) {
 207                 int ret;
 208
 209                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 210                 assert(!ret);
 211                 (void)ret;
 212         }
 213 }
 214
 215 /* Update VLAN's VID/PCP based on input rte_flow_action.
 216  *
 217  * @param[in] action
 218  *   Pointer to struct rte_flow_action.
 219  * @param[out] vlan
 220  *   Pointer to struct rte_vlan_hdr.
 221  */
 222 static void
 223 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 224                          struct rte_vlan_hdr *vlan)
 225 {
 226         uint16_t vlan_tci;
 227         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 228                 vlan_tci =
 229                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 230                                                action->conf)->vlan_pcp;
 231                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 232                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 233                 vlan->vlan_tci |= vlan_tci;
 234         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 235                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 236                 vlan->vlan_tci |= rte_be_to_cpu_16
 237                     (((const struct rte_flow_action_of_set_vlan_vid *)
 238                                              action->conf)->vlan_vid);
 239         }
 240 }
 241
 242 /**
 243  * Fetch 1, 2, 3 or 4 byte field from the byte array
 244  * and return as unsigned integer in host-endian format.
 245  *
 246  * @param[in] data
 247  *   Pointer to data array.
 248  * @param[in] size
 249  *   Size of field to extract.
 250  *
 251  * @return
 252  *   converted field in host endian format.
 253  */
 254 static inline uint32_t
 255 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 256 {
 257         uint32_t ret;
 258
 259         switch (size) {
 260         case 1:
 261                 ret = *data;
 262                 break;
 263         case 2:
 264                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 265                 break;
 266         case 3:
 267                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 268                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 269                 break;
 270         case 4:
 271                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 272                 break;
 273         default:
 274                 assert(false);
 275                 ret = 0;
 276                 break;
 277         }
 278         return ret;
 279 }
 280
 281 /**
 282  * Convert modify-header action to DV specification.
 283  *
 284  * Data length of each action is determined by provided field description
 285  * and the item mask. Data bit offset and width of each action is determined
 286  * by provided item mask.
 287  *
 288  * @param[in] item
 289  *   Pointer to item specification.
 290  * @param[in] field
 291  *   Pointer to field modification information.
 292  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 293  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 294  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 295  * @param[in] dcopy
 296  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 297  *   Negative offset value sets the same offset as source offset.
 298  *   size field is ignored, value is taken from source field.
 299  * @param[in,out] resource
 300  *   Pointer to the modify-header resource.
 301  * @param[in] type
 302  *   Type of modification.
 303  * @param[out] error
 304  *   Pointer to the error structure.
 305  *
 306  * @return
 307  *   0 on success, a negative errno value otherwise and rte_errno is set.
 308  */
 309 static int
 310 flow_dv_convert_modify_action(struct rte_flow_item *item,
 311                               struct field_modify_info *field,
 312                               struct field_modify_info *dcopy,
 313                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 314                               uint32_t type, struct rte_flow_error *error)
 315 {
 316         uint32_t i = resource->actions_num;
 317         struct mlx5_modification_cmd *actions = resource->actions;
 318
 319         /*
 320          * The item and mask are provided in big-endian format.
 321          * The fields should be presented as in big-endian format either.
 322          * Mask must be always present, it defines the actual field width.
 323          */
 324         assert(item->mask);
 325         assert(field->size);
 326         do {
 327                 unsigned int size_b;
 328                 unsigned int off_b;
 329                 uint32_t mask;
 330                 uint32_t data;
 331
 332                 if (i >= MLX5_MODIFY_NUM)
 333                         return rte_flow_error_set(error, EINVAL,
 334                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 335                                  "too many items to modify");
 336                 /* Fetch variable byte size mask from the array. */
 337                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 338                                            field->offset, field->size);
 339                 if (!mask) {
 340                         ++field;
 341                         continue;
 342                 }
 343                 /* Deduce actual data width in bits from mask value. */
 344                 off_b = rte_bsf32(mask);
 345                 size_b = sizeof(uint32_t) * CHAR_BIT -
 346                          off_b - __builtin_clz(mask);
 347                 assert(size_b);
 348                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 349                 actions[i].action_type = type;
 350                 actions[i].field = field->id;
 351                 actions[i].offset = off_b;
 352                 actions[i].length = size_b;
 353                 /* Convert entire record to expected big-endian format. */
 354                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 355                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 356                         assert(dcopy);
 357                         actions[i].dst_field = dcopy->id;
 358                         actions[i].dst_offset =
 359                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 360                         /* Convert entire record to big-endian format. */
 361                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 362                 } else {
 363                         assert(item->spec);
 364                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 365                                                    field->offset, field->size);
 366                         /* Shift out the trailing masked bits from data. */
 367                         data = (data & mask) >> off_b;
 368                         actions[i].data1 = rte_cpu_to_be_32(data);
 369                 }
 370                 ++i;
 371                 ++field;
 372         } while (field->size);
 373         resource->actions_num = i;
 374         if (!resource->actions_num)
 375                 return rte_flow_error_set(error, EINVAL,
 376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 377                                           "invalid modification flow item");
 378         return 0;
 379 }
 380
 381 /**
 382  * Convert modify-header set IPv4 address action to DV specification.
 383  *
 384  * @param[in,out] resource
 385  *   Pointer to the modify-header resource.
 386  * @param[in] action
 387  *   Pointer to action specification.
 388  * @param[out] error
 389  *   Pointer to the error structure.
 390  *
 391  * @return
 392  *   0 on success, a negative errno value otherwise and rte_errno is set.
 393  */
 394 static int
 395 flow_dv_convert_action_modify_ipv4
 396                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 397                          const struct rte_flow_action *action,
 398                          struct rte_flow_error *error)
 399 {
 400         const struct rte_flow_action_set_ipv4 *conf =
 401                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 402         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 403         struct rte_flow_item_ipv4 ipv4;
 404         struct rte_flow_item_ipv4 ipv4_mask;
 405
 406         memset(&ipv4, 0, sizeof(ipv4));
 407         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 408         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 409                 ipv4.hdr.src_addr = conf->ipv4_addr;
 410                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 411         } else {
 412                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 413                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 414         }
 415         item.spec = &ipv4;
 416         item.mask = &ipv4_mask;
 417         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 418                                              MLX5_MODIFICATION_TYPE_SET, error);
 419 }
 420
 421 /**
 422  * Convert modify-header set IPv6 address action to DV specification.
 423  *
 424  * @param[in,out] resource
 425  *   Pointer to the modify-header resource.
 426  * @param[in] action
 427  *   Pointer to action specification.
 428  * @param[out] error
 429  *   Pointer to the error structure.
 430  *
 431  * @return
 432  *   0 on success, a negative errno value otherwise and rte_errno is set.
 433  */
 434 static int
 435 flow_dv_convert_action_modify_ipv6
 436                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 437                          const struct rte_flow_action *action,
 438                          struct rte_flow_error *error)
 439 {
 440         const struct rte_flow_action_set_ipv6 *conf =
 441                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 442         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 443         struct rte_flow_item_ipv6 ipv6;
 444         struct rte_flow_item_ipv6 ipv6_mask;
 445
 446         memset(&ipv6, 0, sizeof(ipv6));
 447         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 448         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 449                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 450                        sizeof(ipv6.hdr.src_addr));
 451                 memcpy(&ipv6_mask.hdr.src_addr,
 452                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 453                        sizeof(ipv6.hdr.src_addr));
 454         } else {
 455                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 456                        sizeof(ipv6.hdr.dst_addr));
 457                 memcpy(&ipv6_mask.hdr.dst_addr,
 458                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 459                        sizeof(ipv6.hdr.dst_addr));
 460         }
 461         item.spec = &ipv6;
 462         item.mask = &ipv6_mask;
 463         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 464                                              MLX5_MODIFICATION_TYPE_SET, error);
 465 }
 466
 467 /**
 468  * Convert modify-header set MAC address action to DV specification.
 469  *
 470  * @param[in,out] resource
 471  *   Pointer to the modify-header resource.
 472  * @param[in] action
 473  *   Pointer to action specification.
 474  * @param[out] error
 475  *   Pointer to the error structure.
 476  *
 477  * @return
 478  *   0 on success, a negative errno value otherwise and rte_errno is set.
 479  */
 480 static int
 481 flow_dv_convert_action_modify_mac
 482                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 483                          const struct rte_flow_action *action,
 484                          struct rte_flow_error *error)
 485 {
 486         const struct rte_flow_action_set_mac *conf =
 487                 (const struct rte_flow_action_set_mac *)(action->conf);
 488         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 489         struct rte_flow_item_eth eth;
 490         struct rte_flow_item_eth eth_mask;
 491
 492         memset(&eth, 0, sizeof(eth));
 493         memset(&eth_mask, 0, sizeof(eth_mask));
 494         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 495                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 496                        sizeof(eth.src.addr_bytes));
 497                 memcpy(&eth_mask.src.addr_bytes,
 498                        &rte_flow_item_eth_mask.src.addr_bytes,
 499                        sizeof(eth_mask.src.addr_bytes));
 500         } else {
 501                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 502                        sizeof(eth.dst.addr_bytes));
 503                 memcpy(&eth_mask.dst.addr_bytes,
 504                        &rte_flow_item_eth_mask.dst.addr_bytes,
 505                        sizeof(eth_mask.dst.addr_bytes));
 506         }
 507         item.spec = &eth;
 508         item.mask = &eth_mask;
 509         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 510                                              MLX5_MODIFICATION_TYPE_SET, error);
 511 }
 512
 513 /**
 514  * Convert modify-header set VLAN VID action to DV specification.
 515  *
 516  * @param[in,out] resource
 517  *   Pointer to the modify-header resource.
 518  * @param[in] action
 519  *   Pointer to action specification.
 520  * @param[out] error
 521  *   Pointer to the error structure.
 522  *
 523  * @return
 524  *   0 on success, a negative errno value otherwise and rte_errno is set.
 525  */
 526 static int
 527 flow_dv_convert_action_modify_vlan_vid
 528                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 529                          const struct rte_flow_action *action,
 530                          struct rte_flow_error *error)
 531 {
 532         const struct rte_flow_action_of_set_vlan_vid *conf =
 533                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 534         int i = resource->actions_num;
 535         struct mlx5_modification_cmd *actions = &resource->actions[i];
 536         struct field_modify_info *field = modify_vlan_out_first_vid;
 537
 538         if (i >= MLX5_MODIFY_NUM)
 539                 return rte_flow_error_set(error, EINVAL,
 540                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 541                          "too many items to modify");
 542         actions[i].action_type = MLX5_MODIFICATION_TYPE_SET;
 543         actions[i].field = field->id;
 544         actions[i].length = field->size;
 545         actions[i].offset = field->offset;
 546         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 547         actions[i].data1 = conf->vlan_vid;
 548         actions[i].data1 = actions[i].data1 << 16;
 549         resource->actions_num = ++i;
 550         return 0;
 551 }
 552
 553 /**
 554  * Convert modify-header set TP action to DV specification.
 555  *
 556  * @param[in,out] resource
 557  *   Pointer to the modify-header resource.
 558  * @param[in] action
 559  *   Pointer to action specification.
 560  * @param[in] items
 561  *   Pointer to rte_flow_item objects list.
 562  * @param[in] attr
 563  *   Pointer to flow attributes structure.
 564  * @param[out] error
 565  *   Pointer to the error structure.
 566  *
 567  * @return
 568  *   0 on success, a negative errno value otherwise and rte_errno is set.
 569  */
 570 static int
 571 flow_dv_convert_action_modify_tp
 572                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 573                          const struct rte_flow_action *action,
 574                          const struct rte_flow_item *items,
 575                          union flow_dv_attr *attr,
 576                          struct rte_flow_error *error)
 577 {
 578         const struct rte_flow_action_set_tp *conf =
 579                 (const struct rte_flow_action_set_tp *)(action->conf);
 580         struct rte_flow_item item;
 581         struct rte_flow_item_udp udp;
 582         struct rte_flow_item_udp udp_mask;
 583         struct rte_flow_item_tcp tcp;
 584         struct rte_flow_item_tcp tcp_mask;
 585         struct field_modify_info *field;
 586
 587         if (!attr->valid)
 588                 flow_dv_attr_init(items, attr);
 589         if (attr->udp) {
 590                 memset(&udp, 0, sizeof(udp));
 591                 memset(&udp_mask, 0, sizeof(udp_mask));
 592                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 593                         udp.hdr.src_port = conf->port;
 594                         udp_mask.hdr.src_port =
 595                                         rte_flow_item_udp_mask.hdr.src_port;
 596                 } else {
 597                         udp.hdr.dst_port = conf->port;
 598                         udp_mask.hdr.dst_port =
 599                                         rte_flow_item_udp_mask.hdr.dst_port;
 600                 }
 601                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 602                 item.spec = &udp;
 603                 item.mask = &udp_mask;
 604                 field = modify_udp;
 605         }
 606         if (attr->tcp) {
 607                 memset(&tcp, 0, sizeof(tcp));
 608                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 609                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 610                         tcp.hdr.src_port = conf->port;
 611                         tcp_mask.hdr.src_port =
 612                                         rte_flow_item_tcp_mask.hdr.src_port;
 613                 } else {
 614                         tcp.hdr.dst_port = conf->port;
 615                         tcp_mask.hdr.dst_port =
 616                                         rte_flow_item_tcp_mask.hdr.dst_port;
 617                 }
 618                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 619                 item.spec = &tcp;
 620                 item.mask = &tcp_mask;
 621                 field = modify_tcp;
 622         }
 623         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 624                                              MLX5_MODIFICATION_TYPE_SET, error);
 625 }
 626
 627 /**
 628  * Convert modify-header set TTL action to DV specification.
 629  *
 630  * @param[in,out] resource
 631  *   Pointer to the modify-header resource.
 632  * @param[in] action
 633  *   Pointer to action specification.
 634  * @param[in] items
 635  *   Pointer to rte_flow_item objects list.
 636  * @param[in] attr
 637  *   Pointer to flow attributes structure.
 638  * @param[out] error
 639  *   Pointer to the error structure.
 640  *
 641  * @return
 642  *   0 on success, a negative errno value otherwise and rte_errno is set.
 643  */
 644 static int
 645 flow_dv_convert_action_modify_ttl
 646                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 647                          const struct rte_flow_action *action,
 648                          const struct rte_flow_item *items,
 649                          union flow_dv_attr *attr,
 650                          struct rte_flow_error *error)
 651 {
 652         const struct rte_flow_action_set_ttl *conf =
 653                 (const struct rte_flow_action_set_ttl *)(action->conf);
 654         struct rte_flow_item item;
 655         struct rte_flow_item_ipv4 ipv4;
 656         struct rte_flow_item_ipv4 ipv4_mask;
 657         struct rte_flow_item_ipv6 ipv6;
 658         struct rte_flow_item_ipv6 ipv6_mask;
 659         struct field_modify_info *field;
 660
 661         if (!attr->valid)
 662                 flow_dv_attr_init(items, attr);
 663         if (attr->ipv4) {
 664                 memset(&ipv4, 0, sizeof(ipv4));
 665                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 666                 ipv4.hdr.time_to_live = conf->ttl_value;
 667                 ipv4_mask.hdr.time_to_live = 0xFF;
 668                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 669                 item.spec = &ipv4;
 670                 item.mask = &ipv4_mask;
 671                 field = modify_ipv4;
 672         }
 673         if (attr->ipv6) {
 674                 memset(&ipv6, 0, sizeof(ipv6));
 675                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 676                 ipv6.hdr.hop_limits = conf->ttl_value;
 677                 ipv6_mask.hdr.hop_limits = 0xFF;
 678                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 679                 item.spec = &ipv6;
 680                 item.mask = &ipv6_mask;
 681                 field = modify_ipv6;
 682         }
 683         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 684                                              MLX5_MODIFICATION_TYPE_SET, error);
 685 }
 686
 687 /**
 688  * Convert modify-header decrement TTL action to DV specification.
 689  *
 690  * @param[in,out] resource
 691  *   Pointer to the modify-header resource.
 692  * @param[in] action
 693  *   Pointer to action specification.
 694  * @param[in] items
 695  *   Pointer to rte_flow_item objects list.
 696  * @param[in] attr
 697  *   Pointer to flow attributes structure.
 698  * @param[out] error
 699  *   Pointer to the error structure.
 700  *
 701  * @return
 702  *   0 on success, a negative errno value otherwise and rte_errno is set.
 703  */
 704 static int
 705 flow_dv_convert_action_modify_dec_ttl
 706                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 707                          const struct rte_flow_item *items,
 708                          union flow_dv_attr *attr,
 709                          struct rte_flow_error *error)
 710 {
 711         struct rte_flow_item item;
 712         struct rte_flow_item_ipv4 ipv4;
 713         struct rte_flow_item_ipv4 ipv4_mask;
 714         struct rte_flow_item_ipv6 ipv6;
 715         struct rte_flow_item_ipv6 ipv6_mask;
 716         struct field_modify_info *field;
 717
 718         if (!attr->valid)
 719                 flow_dv_attr_init(items, attr);
 720         if (attr->ipv4) {
 721                 memset(&ipv4, 0, sizeof(ipv4));
 722                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 723                 ipv4.hdr.time_to_live = 0xFF;
 724                 ipv4_mask.hdr.time_to_live = 0xFF;
 725                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 726                 item.spec = &ipv4;
 727                 item.mask = &ipv4_mask;
 728                 field = modify_ipv4;
 729         }
 730         if (attr->ipv6) {
 731                 memset(&ipv6, 0, sizeof(ipv6));
 732                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 733                 ipv6.hdr.hop_limits = 0xFF;
 734                 ipv6_mask.hdr.hop_limits = 0xFF;
 735                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 736                 item.spec = &ipv6;
 737                 item.mask = &ipv6_mask;
 738                 field = modify_ipv6;
 739         }
 740         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 741                                              MLX5_MODIFICATION_TYPE_ADD, error);
 742 }
 743
 744 /**
 745  * Convert modify-header increment/decrement TCP Sequence number
 746  * to DV specification.
 747  *
 748  * @param[in,out] resource
 749  *   Pointer to the modify-header resource.
 750  * @param[in] action
 751  *   Pointer to action specification.
 752  * @param[out] error
 753  *   Pointer to the error structure.
 754  *
 755  * @return
 756  *   0 on success, a negative errno value otherwise and rte_errno is set.
 757  */
 758 static int
 759 flow_dv_convert_action_modify_tcp_seq
 760                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 761                          const struct rte_flow_action *action,
 762                          struct rte_flow_error *error)
 763 {
 764         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 765         uint64_t value = rte_be_to_cpu_32(*conf);
 766         struct rte_flow_item item;
 767         struct rte_flow_item_tcp tcp;
 768         struct rte_flow_item_tcp tcp_mask;
 769
 770         memset(&tcp, 0, sizeof(tcp));
 771         memset(&tcp_mask, 0, sizeof(tcp_mask));
 772         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 773                 /*
 774                  * The HW has no decrement operation, only increment operation.
 775                  * To simulate decrement X from Y using increment operation
 776                  * we need to add UINT32_MAX X times to Y.
 777                  * Each adding of UINT32_MAX decrements Y by 1.
 778                  */
 779                 value *= UINT32_MAX;
 780         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 781         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 782         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 783         item.spec = &tcp;
 784         item.mask = &tcp_mask;
 785         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 786                                              MLX5_MODIFICATION_TYPE_ADD, error);
 787 }
 788
 789 /**
 790  * Convert modify-header increment/decrement TCP Acknowledgment number
 791  * to DV specification.
 792  *
 793  * @param[in,out] resource
 794  *   Pointer to the modify-header resource.
 795  * @param[in] action
 796  *   Pointer to action specification.
 797  * @param[out] error
 798  *   Pointer to the error structure.
 799  *
 800  * @return
 801  *   0 on success, a negative errno value otherwise and rte_errno is set.
 802  */
 803 static int
 804 flow_dv_convert_action_modify_tcp_ack
 805                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 806                          const struct rte_flow_action *action,
 807                          struct rte_flow_error *error)
 808 {
 809         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 810         uint64_t value = rte_be_to_cpu_32(*conf);
 811         struct rte_flow_item item;
 812         struct rte_flow_item_tcp tcp;
 813         struct rte_flow_item_tcp tcp_mask;
 814
 815         memset(&tcp, 0, sizeof(tcp));
 816         memset(&tcp_mask, 0, sizeof(tcp_mask));
 817         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 818                 /*
 819                  * The HW has no decrement operation, only increment operation.
 820                  * To simulate decrement X from Y using increment operation
 821                  * we need to add UINT32_MAX X times to Y.
 822                  * Each adding of UINT32_MAX decrements Y by 1.
 823                  */
 824                 value *= UINT32_MAX;
 825         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 826         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 827         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 828         item.spec = &tcp;
 829         item.mask = &tcp_mask;
 830         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 831                                              MLX5_MODIFICATION_TYPE_ADD, error);
 832 }
 833
 834 static enum mlx5_modification_field reg_to_field[] = {
 835         [REG_NONE] = MLX5_MODI_OUT_NONE,
 836         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 837         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 838         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 839         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 840         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 841         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 842         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 843         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 844         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 845         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 846 };
 847
 848 /**
 849  * Convert register set to DV specification.
 850  *
 851  * @param[in,out] resource
 852  *   Pointer to the modify-header resource.
 853  * @param[in] action
 854  *   Pointer to action specification.
 855  * @param[out] error
 856  *   Pointer to the error structure.
 857  *
 858  * @return
 859  *   0 on success, a negative errno value otherwise and rte_errno is set.
 860  */
 861 static int
 862 flow_dv_convert_action_set_reg
 863                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 864                          const struct rte_flow_action *action,
 865                          struct rte_flow_error *error)
 866 {
 867         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 868         struct mlx5_modification_cmd *actions = resource->actions;
 869         uint32_t i = resource->actions_num;
 870
 871         if (i >= MLX5_MODIFY_NUM)
 872                 return rte_flow_error_set(error, EINVAL,
 873                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 874                                           "too many items to modify");
 875         actions[i].action_type = MLX5_MODIFICATION_TYPE_SET;
 876         actions[i].field = reg_to_field[conf->id];
 877         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 878         actions[i].data1 = conf->data;
 879         ++i;
 880         resource->actions_num = i;
 881         if (!resource->actions_num)
 882                 return rte_flow_error_set(error, EINVAL,
 883                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 884                                           "invalid modification flow item");
 885         return 0;
 886 }
 887
 888 /**
 889  * Convert internal COPY_REG action to DV specification.
 890  *
 891  * @param[in] dev
 892  *   Pointer to the rte_eth_dev structure.
 893  * @param[in,out] res
 894  *   Pointer to the modify-header resource.
 895  * @param[in] action
 896  *   Pointer to action specification.
 897  * @param[out] error
 898  *   Pointer to the error structure.
 899  *
 900  * @return
 901  *   0 on success, a negative errno value otherwise and rte_errno is set.
 902  */
 903 static int
 904 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev  __rte_unused,
 905                                  struct mlx5_flow_dv_modify_hdr_resource *res,
 906                                  const struct rte_flow_action *action,
 907                                  struct rte_flow_error *error)
 908 {
 909         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
 910         uint32_t mask = RTE_BE32(UINT32_MAX);
 911         struct rte_flow_item item = {
 912                 .spec = NULL,
 913                 .mask = &mask,
 914         };
 915         struct field_modify_info reg_src[] = {
 916                 {4, 0, reg_to_field[conf->src]},
 917                 {0, 0, 0},
 918         };
 919         struct field_modify_info reg_dst = {
 920                 .offset = (uint32_t)-1, /* Same as src. */
 921                 .id = reg_to_field[conf->dst],
 922         };
 923         return flow_dv_convert_modify_action(&item,
 924                                              reg_src, &reg_dst, res,
 925                                              MLX5_MODIFICATION_TYPE_COPY,
 926                                              error);
 927 }
 928
 929 /**
 930  * Validate META item.
 931  *
 932  * @param[in] dev
 933  *   Pointer to the rte_eth_dev structure.
 934  * @param[in] item
 935  *   Item specification.
 936  * @param[in] attr
 937  *   Attributes of flow that includes this item.
 938  * @param[out] error
 939  *   Pointer to error structure.
 940  *
 941  * @return
 942  *   0 on success, a negative errno value otherwise and rte_errno is set.
 943  */
 944 static int
 945 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
 946                            const struct rte_flow_item *item,
 947                            const struct rte_flow_attr *attr,
 948                            struct rte_flow_error *error)
 949 {
 950         const struct rte_flow_item_meta *spec = item->spec;
 951         const struct rte_flow_item_meta *mask = item->mask;
 952         const struct rte_flow_item_meta nic_mask = {
 953                 .data = UINT32_MAX
 954         };
 955         int ret;
 956
 957         if (!spec)
 958                 return rte_flow_error_set(error, EINVAL,
 959                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 960                                           item->spec,
 961                                           "data cannot be empty");
 962         if (!spec->data)
 963                 return rte_flow_error_set(error, EINVAL,
 964                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 965                                           NULL,
 966                                           "data cannot be zero");
 967         if (!mask)
 968                 mask = &rte_flow_item_meta_mask;
 969         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 970                                         (const uint8_t *)&nic_mask,
 971                                         sizeof(struct rte_flow_item_meta),
 972                                         error);
 973         if (ret < 0)
 974                 return ret;
 975         if (attr->ingress)
 976                 return rte_flow_error_set(error, ENOTSUP,
 977                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 978                                           NULL,
 979                                           "pattern not supported for ingress");
 980         return 0;
 981 }
 982
 983 /**
 984  * Validate vport item.
 985  *
 986  * @param[in] dev
 987  *   Pointer to the rte_eth_dev structure.
 988  * @param[in] item
 989  *   Item specification.
 990  * @param[in] attr
 991  *   Attributes of flow that includes this item.
 992  * @param[in] item_flags
 993  *   Bit-fields that holds the items detected until now.
 994  * @param[out] error
 995  *   Pointer to error structure.
 996  *
 997  * @return
 998  *   0 on success, a negative errno value otherwise and rte_errno is set.
 999  */
1000 static int
1001 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1002                               const struct rte_flow_item *item,
1003                               const struct rte_flow_attr *attr,
1004                               uint64_t item_flags,
1005                               struct rte_flow_error *error)
1006 {
1007         const struct rte_flow_item_port_id *spec = item->spec;
1008         const struct rte_flow_item_port_id *mask = item->mask;
1009         const struct rte_flow_item_port_id switch_mask = {
1010                         .id = 0xffffffff,
1011         };
1012         struct mlx5_priv *esw_priv;
1013         struct mlx5_priv *dev_priv;
1014         int ret;
1015
1016         if (!attr->transfer)
1017                 return rte_flow_error_set(error, EINVAL,
1018                                           RTE_FLOW_ERROR_TYPE_ITEM,
1019                                           NULL,
1020                                           "match on port id is valid only"
1021                                           " when transfer flag is enabled");
1022         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1023                 return rte_flow_error_set(error, ENOTSUP,
1024                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1025                                           "multiple source ports are not"
1026                                           " supported");
1027         if (!mask)
1028                 mask = &switch_mask;
1029         if (mask->id != 0xffffffff)
1030                 return rte_flow_error_set(error, ENOTSUP,
1031                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1032                                            mask,
1033                                            "no support for partial mask on"
1034                                            " \"id\" field");
1035         ret = mlx5_flow_item_acceptable
1036                                 (item, (const uint8_t *)mask,
1037                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1038                                  sizeof(struct rte_flow_item_port_id),
1039                                  error);
1040         if (ret)
1041                 return ret;
1042         if (!spec)
1043                 return 0;
1044         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1045         if (!esw_priv)
1046                 return rte_flow_error_set(error, rte_errno,
1047                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1048                                           "failed to obtain E-Switch info for"
1049                                           " port");
1050         dev_priv = mlx5_dev_to_eswitch_info(dev);
1051         if (!dev_priv)
1052                 return rte_flow_error_set(error, rte_errno,
1053                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1054                                           NULL,
1055                                           "failed to obtain E-Switch info");
1056         if (esw_priv->domain_id != dev_priv->domain_id)
1057                 return rte_flow_error_set(error, EINVAL,
1058                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1059                                           "cannot match on a port from a"
1060                                           " different E-Switch");
1061         return 0;
1062 }
1063
1064 /**
1065  * Validate the pop VLAN action.
1066  *
1067  * @param[in] dev
1068  *   Pointer to the rte_eth_dev structure.
1069  * @param[in] action_flags
1070  *   Holds the actions detected until now.
1071  * @param[in] action
1072  *   Pointer to the pop vlan action.
1073  * @param[in] item_flags
1074  *   The items found in this flow rule.
1075  * @param[in] attr
1076  *   Pointer to flow attributes.
1077  * @param[out] error
1078  *   Pointer to error structure.
1079  *
1080  * @return
1081  *   0 on success, a negative errno value otherwise and rte_errno is set.
1082  */
1083 static int
1084 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1085                                  uint64_t action_flags,
1086                                  const struct rte_flow_action *action,
1087                                  uint64_t item_flags,
1088                                  const struct rte_flow_attr *attr,
1089                                  struct rte_flow_error *error)
1090 {
1091         struct mlx5_priv *priv = dev->data->dev_private;
1092
1093         (void)action;
1094         (void)attr;
1095         if (!priv->sh->pop_vlan_action)
1096                 return rte_flow_error_set(error, ENOTSUP,
1097                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1098                                           NULL,
1099                                           "pop vlan action is not supported");
1100         /*
1101          * Check for inconsistencies:
1102          *  fail strip_vlan in a flow that matches packets without VLAN tags.
1103          *  fail strip_vlan in a flow that matches packets without explicitly a
1104          *  matching on VLAN tag ?
1105          */
1106         if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN)
1107                 return rte_flow_error_set(error, ENOTSUP,
1108                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1109                                           NULL,
1110                                           "no support for multiple vlan pop "
1111                                           "actions");
1112         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1113                 return rte_flow_error_set(error, ENOTSUP,
1114                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1115                                           NULL,
1116                                           "cannot pop vlan without a "
1117                                           "match on (outer) vlan in the flow");
1118         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1119                 return rte_flow_error_set(error, EINVAL,
1120                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1121                                           "wrong action order, port_id should "
1122                                           "be after pop VLAN action");
1123         return 0;
1124 }
1125
1126 /**
1127  * Get VLAN default info from vlan match info.
1128  *
1129  * @param[in] dev
1130  *   Pointer to the rte_eth_dev structure.
1131  * @param[in] item
1132  *   the list of item specifications.
1133  * @param[out] vlan
1134  *   pointer VLAN info to fill to.
1135  * @param[out] error
1136  *   Pointer to error structure.
1137  *
1138  * @return
1139  *   0 on success, a negative errno value otherwise and rte_errno is set.
1140  */
1141 static void
1142 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1143                                   struct rte_vlan_hdr *vlan)
1144 {
1145         const struct rte_flow_item_vlan nic_mask = {
1146                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1147                                 MLX5DV_FLOW_VLAN_VID_MASK),
1148                 .inner_type = RTE_BE16(0xffff),
1149         };
1150
1151         if (items == NULL)
1152                 return;
1153         for (; items->type != RTE_FLOW_ITEM_TYPE_END &&
1154                items->type != RTE_FLOW_ITEM_TYPE_VLAN; items++)
1155                 ;
1156         if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1157                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1158                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1159
1160                 if (!vlan_m)
1161                         vlan_m = &nic_mask;
1162                 /* Only full match values are accepted */
1163                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1164                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1165                         vlan->vlan_tci &= MLX5DV_FLOW_VLAN_PCP_MASK;
1166                         vlan->vlan_tci |=
1167                                 rte_be_to_cpu_16(vlan_v->tci &
1168                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1169                 }
1170                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1171                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1172                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1173                         vlan->vlan_tci |=
1174                                 rte_be_to_cpu_16(vlan_v->tci &
1175                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1176                 }
1177                 if (vlan_m->inner_type == nic_mask.inner_type)
1178                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1179                                                            vlan_m->inner_type);
1180         }
1181 }
1182
1183 /**
1184  * Validate the push VLAN action.
1185  *
1186  * @param[in] action_flags
1187  *   Holds the actions detected until now.
1188  * @param[in] action
1189  *   Pointer to the encap action.
1190  * @param[in] attr
1191  *   Pointer to flow attributes
1192  * @param[out] error
1193  *   Pointer to error structure.
1194  *
1195  * @return
1196  *   0 on success, a negative errno value otherwise and rte_errno is set.
1197  */
1198 static int
1199 flow_dv_validate_action_push_vlan(uint64_t action_flags,
1200                                   uint64_t item_flags,
1201                                   const struct rte_flow_action *action,
1202                                   const struct rte_flow_attr *attr,
1203                                   struct rte_flow_error *error)
1204 {
1205         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1206
1207         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1208             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1209                 return rte_flow_error_set(error, EINVAL,
1210                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1211                                           "invalid vlan ethertype");
1212         if (action_flags &
1213                 (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1214                 return rte_flow_error_set(error, ENOTSUP,
1215                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1216                                           "no support for multiple VLAN "
1217                                           "actions");
1218         if (!mlx5_flow_find_action
1219                         (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) &&
1220             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1221                 return rte_flow_error_set(error, ENOTSUP,
1222                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
1223                                 "push VLAN needs to match on VLAN in order to "
1224                                 "get VLAN VID information because there is "
1225                                 "no followed set VLAN VID action");
1226         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1227                 return rte_flow_error_set(error, EINVAL,
1228                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1229                                           "wrong action order, port_id should "
1230                                           "be after push VLAN");
1231         (void)attr;
1232         return 0;
1233 }
1234
1235 /**
1236  * Validate the set VLAN PCP.
1237  *
1238  * @param[in] action_flags
1239  *   Holds the actions detected until now.
1240  * @param[in] actions
1241  *   Pointer to the list of actions remaining in the flow rule.
1242  * @param[in] attr
1243  *   Pointer to flow attributes
1244  * @param[out] error
1245  *   Pointer to error structure.
1246  *
1247  * @return
1248  *   0 on success, a negative errno value otherwise and rte_errno is set.
1249  */
1250 static int
1251 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
1252                                      const struct rte_flow_action actions[],
1253                                      struct rte_flow_error *error)
1254 {
1255         const struct rte_flow_action *action = actions;
1256         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
1257
1258         if (conf->vlan_pcp > 7)
1259                 return rte_flow_error_set(error, EINVAL,
1260                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1261                                           "VLAN PCP value is too big");
1262         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
1263                 return rte_flow_error_set(error, ENOTSUP,
1264                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1265                                           "set VLAN PCP action must follow "
1266                                           "the push VLAN action");
1267         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
1268                 return rte_flow_error_set(error, ENOTSUP,
1269                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1270                                           "Multiple VLAN PCP modification are "
1271                                           "not supported");
1272         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1273                 return rte_flow_error_set(error, EINVAL,
1274                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1275                                           "wrong action order, port_id should "
1276                                           "be after set VLAN PCP");
1277         return 0;
1278 }
1279
1280 /**
1281  * Validate the set VLAN VID.
1282  *
1283  * @param[in] item_flags
1284  *   Holds the items detected in this rule.
1285  * @param[in] actions
1286  *   Pointer to the list of actions remaining in the flow rule.
1287  * @param[in] attr
1288  *   Pointer to flow attributes
1289  * @param[out] error
1290  *   Pointer to error structure.
1291  *
1292  * @return
1293  *   0 on success, a negative errno value otherwise and rte_errno is set.
1294  */
1295 static int
1296 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
1297                                      uint64_t action_flags,
1298                                      const struct rte_flow_action actions[],
1299                                      struct rte_flow_error *error)
1300 {
1301         const struct rte_flow_action *action = actions;
1302         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
1303
1304         if (conf->vlan_vid > RTE_BE16(0xFFE))
1305                 return rte_flow_error_set(error, EINVAL,
1306                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1307                                           "VLAN VID value is too big");
1308         /* there is an of_push_vlan action before us */
1309         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) {
1310                 if (mlx5_flow_find_action(actions + 1,
1311                                           RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID))
1312                         return rte_flow_error_set(error, ENOTSUP,
1313                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1314                                         "Multiple VLAN VID modifications are "
1315                                         "not supported");
1316                 else
1317                         return 0;
1318         }
1319
1320         /*
1321          * Action is on an existing VLAN header:
1322          *    Need to verify this is a single modify CID action.
1323          *   Rule mast include a match on outer VLAN.
1324          */
1325         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
1326                 return rte_flow_error_set(error, ENOTSUP,
1327                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1328                                           "Multiple VLAN VID modifications are "
1329                                           "not supported");
1330         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1331                 return rte_flow_error_set(error, EINVAL,
1332                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1333                                           "match on VLAN is required in order "
1334                                           "to set VLAN VID");
1335         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1336                 return rte_flow_error_set(error, EINVAL,
1337                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1338                                           "wrong action order, port_id should "
1339                                           "be after set VLAN VID");
1340         return 0;
1341 }
1342
1343 /**
1344  * Validate count action.
1345  *
1346  * @param[in] dev
1347  *   device otr.
1348  * @param[out] error
1349  *   Pointer to error structure.
1350  *
1351  * @return
1352  *   0 on success, a negative errno value otherwise and rte_errno is set.
1353  */
1354 static int
1355 flow_dv_validate_action_count(struct rte_eth_dev *dev,
1356                               struct rte_flow_error *error)
1357 {
1358         struct mlx5_priv *priv = dev->data->dev_private;
1359
1360         if (!priv->config.devx)
1361                 goto notsup_err;
1362 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
1363         return 0;
1364 #endif
1365 notsup_err:
1366         return rte_flow_error_set
1367                       (error, ENOTSUP,
1368                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1369                        NULL,
1370                        "count action not supported");
1371 }
1372
1373 /**
1374  * Validate the L2 encap action.
1375  *
1376  * @param[in] action_flags
1377  *   Holds the actions detected until now.
1378  * @param[in] action
1379  *   Pointer to the encap action.
1380  * @param[in] attr
1381  *   Pointer to flow attributes
1382  * @param[out] error
1383  *   Pointer to error structure.
1384  *
1385  * @return
1386  *   0 on success, a negative errno value otherwise and rte_errno is set.
1387  */
1388 static int
1389 flow_dv_validate_action_l2_encap(uint64_t action_flags,
1390                                  const struct rte_flow_action *action,
1391                                  const struct rte_flow_attr *attr,
1392                                  struct rte_flow_error *error)
1393 {
1394         if (!(action->conf))
1395                 return rte_flow_error_set(error, EINVAL,
1396                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1397                                           "configuration cannot be null");
1398         if (action_flags & MLX5_FLOW_ACTION_DROP)
1399                 return rte_flow_error_set(error, EINVAL,
1400                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1401                                           "can't drop and encap in same flow");
1402         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1403                 return rte_flow_error_set(error, EINVAL,
1404                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1405                                           "can only have a single encap or"
1406                                           " decap action in a flow");
1407         if (!attr->transfer && attr->ingress)
1408                 return rte_flow_error_set(error, ENOTSUP,
1409                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1410                                           NULL,
1411                                           "encap action not supported for "
1412                                           "ingress");
1413         return 0;
1414 }
1415
1416 /**
1417  * Validate the L2 decap action.
1418  *
1419  * @param[in] action_flags
1420  *   Holds the actions detected until now.
1421  * @param[in] attr
1422  *   Pointer to flow attributes
1423  * @param[out] error
1424  *   Pointer to error structure.
1425  *
1426  * @return
1427  *   0 on success, a negative errno value otherwise and rte_errno is set.
1428  */
1429 static int
1430 flow_dv_validate_action_l2_decap(uint64_t action_flags,
1431                                  const struct rte_flow_attr *attr,
1432                                  struct rte_flow_error *error)
1433 {
1434         if (action_flags & MLX5_FLOW_ACTION_DROP)
1435                 return rte_flow_error_set(error, EINVAL,
1436                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1437                                           "can't drop and decap in same flow");
1438         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1439                 return rte_flow_error_set(error, EINVAL,
1440                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1441                                           "can only have a single encap or"
1442                                           " decap action in a flow");
1443         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1444                 return rte_flow_error_set(error, EINVAL,
1445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1446                                           "can't have decap action after"
1447                                           " modify action");
1448         if (attr->egress)
1449                 return rte_flow_error_set(error, ENOTSUP,
1450                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1451                                           NULL,
1452                                           "decap action not supported for "
1453                                           "egress");
1454         return 0;
1455 }
1456
1457 /**
1458  * Validate the raw encap action.
1459  *
1460  * @param[in] action_flags
1461  *   Holds the actions detected until now.
1462  * @param[in] action
1463  *   Pointer to the encap action.
1464  * @param[in] attr
1465  *   Pointer to flow attributes
1466  * @param[out] error
1467  *   Pointer to error structure.
1468  *
1469  * @return
1470  *   0 on success, a negative errno value otherwise and rte_errno is set.
1471  */
1472 static int
1473 flow_dv_validate_action_raw_encap(uint64_t action_flags,
1474                                   const struct rte_flow_action *action,
1475                                   const struct rte_flow_attr *attr,
1476                                   struct rte_flow_error *error)
1477 {
1478         const struct rte_flow_action_raw_encap *raw_encap =
1479                 (const struct rte_flow_action_raw_encap *)action->conf;
1480         if (!(action->conf))
1481                 return rte_flow_error_set(error, EINVAL,
1482                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1483                                           "configuration cannot be null");
1484         if (action_flags & MLX5_FLOW_ACTION_DROP)
1485                 return rte_flow_error_set(error, EINVAL,
1486                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1487                                           "can't drop and encap in same flow");
1488         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1489                 return rte_flow_error_set(error, EINVAL,
1490                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1491                                           "can only have a single encap"
1492                                           " action in a flow");
1493         /* encap without preceding decap is not supported for ingress */
1494         if (!attr->transfer &&  attr->ingress &&
1495             !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
1496                 return rte_flow_error_set(error, ENOTSUP,
1497                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1498                                           NULL,
1499                                           "encap action not supported for "
1500                                           "ingress");
1501         if (!raw_encap->size || !raw_encap->data)
1502                 return rte_flow_error_set(error, EINVAL,
1503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1504                                           "raw encap data cannot be empty");
1505         return 0;
1506 }
1507
1508 /**
1509  * Validate the raw decap action.
1510  *
1511  * @param[in] action_flags
1512  *   Holds the actions detected until now.
1513  * @param[in] action
1514  *   Pointer to the encap action.
1515  * @param[in] attr
1516  *   Pointer to flow attributes
1517  * @param[out] error
1518  *   Pointer to error structure.
1519  *
1520  * @return
1521  *   0 on success, a negative errno value otherwise and rte_errno is set.
1522  */
1523 static int
1524 flow_dv_validate_action_raw_decap(uint64_t action_flags,
1525                                   const struct rte_flow_action *action,
1526                                   const struct rte_flow_attr *attr,
1527                                   struct rte_flow_error *error)
1528 {
1529         if (action_flags & MLX5_FLOW_ACTION_DROP)
1530                 return rte_flow_error_set(error, EINVAL,
1531                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1532                                           "can't drop and decap in same flow");
1533         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1534                 return rte_flow_error_set(error, EINVAL,
1535                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1536                                           "can't have encap action before"
1537                                           " decap action");
1538         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1539                 return rte_flow_error_set(error, EINVAL,
1540                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1541                                           "can only have a single decap"
1542                                           " action in a flow");
1543         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1546                                           "can't have decap action after"
1547                                           " modify action");
1548         /* decap action is valid on egress only if it is followed by encap */
1549         if (attr->egress) {
1550                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1551                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1552                        action++) {
1553                 }
1554                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1555                         return rte_flow_error_set
1556                                         (error, ENOTSUP,
1557                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1558                                          NULL, "decap action not supported"
1559                                          " for egress");
1560         }
1561         return 0;
1562 }
1563
1564 /**
1565  * Find existing encap/decap resource or create and register a new one.
1566  *
1567  * @param dev[in, out]
1568  *   Pointer to rte_eth_dev structure.
1569  * @param[in, out] resource
1570  *   Pointer to encap/decap resource.
1571  * @parm[in, out] dev_flow
1572  *   Pointer to the dev_flow.
1573  * @param[out] error
1574  *   pointer to error structure.
1575  *
1576  * @return
1577  *   0 on success otherwise -errno and errno is set.
1578  */
1579 static int
1580 flow_dv_encap_decap_resource_register
1581                         (struct rte_eth_dev *dev,
1582                          struct mlx5_flow_dv_encap_decap_resource *resource,
1583                          struct mlx5_flow *dev_flow,
1584                          struct rte_flow_error *error)
1585 {
1586         struct mlx5_priv *priv = dev->data->dev_private;
1587         struct mlx5_ibv_shared *sh = priv->sh;
1588         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1589         struct mlx5dv_dr_domain *domain;
1590
1591         resource->flags = dev_flow->group ? 0 : 1;
1592         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1593                 domain = sh->fdb_domain;
1594         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1595                 domain = sh->rx_domain;
1596         else
1597                 domain = sh->tx_domain;
1598
1599         /* Lookup a matching resource from cache. */
1600         LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1601                 if (resource->reformat_type == cache_resource->reformat_type &&
1602                     resource->ft_type == cache_resource->ft_type &&
1603                     resource->flags == cache_resource->flags &&
1604                     resource->size == cache_resource->size &&
1605                     !memcmp((const void *)resource->buf,
1606                             (const void *)cache_resource->buf,
1607                             resource->size)) {
1608                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1609                                 (void *)cache_resource,
1610                                 rte_atomic32_read(&cache_resource->refcnt));
1611                         rte_atomic32_inc(&cache_resource->refcnt);
1612                         dev_flow->dv.encap_decap = cache_resource;
1613                         return 0;
1614                 }
1615         }
1616         /* Register new encap/decap resource. */
1617         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1618         if (!cache_resource)
1619                 return rte_flow_error_set(error, ENOMEM,
1620                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1621                                           "cannot allocate resource memory");
1622         *cache_resource = *resource;
1623         cache_resource->verbs_action =
1624                 mlx5_glue->dv_create_flow_action_packet_reformat
1625                         (sh->ctx, cache_resource->reformat_type,
1626                          cache_resource->ft_type, domain, cache_resource->flags,
1627                          cache_resource->size,
1628                          (cache_resource->size ? cache_resource->buf : NULL));
1629         if (!cache_resource->verbs_action) {
1630                 rte_free(cache_resource);
1631                 return rte_flow_error_set(error, ENOMEM,
1632                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1633                                           NULL, "cannot create action");
1634         }
1635         rte_atomic32_init(&cache_resource->refcnt);
1636         rte_atomic32_inc(&cache_resource->refcnt);
1637         LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1638         dev_flow->dv.encap_decap = cache_resource;
1639         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1640                 (void *)cache_resource,
1641                 rte_atomic32_read(&cache_resource->refcnt));
1642         return 0;
1643 }
1644
1645 /**
1646  * Find existing table jump resource or create and register a new one.
1647  *
1648  * @param dev[in, out]
1649  *   Pointer to rte_eth_dev structure.
1650  * @param[in, out] resource
1651  *   Pointer to jump table resource.
1652  * @parm[in, out] dev_flow
1653  *   Pointer to the dev_flow.
1654  * @param[out] error
1655  *   pointer to error structure.
1656  *
1657  * @return
1658  *   0 on success otherwise -errno and errno is set.
1659  */
1660 static int
1661 flow_dv_jump_tbl_resource_register
1662                         (struct rte_eth_dev *dev,
1663                          struct mlx5_flow_dv_jump_tbl_resource *resource,
1664                          struct mlx5_flow *dev_flow,
1665                          struct rte_flow_error *error)
1666 {
1667         struct mlx5_priv *priv = dev->data->dev_private;
1668         struct mlx5_ibv_shared *sh = priv->sh;
1669         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1670
1671         /* Lookup a matching resource from cache. */
1672         LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1673                 if (resource->tbl == cache_resource->tbl) {
1674                         DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1675                                 (void *)cache_resource,
1676                                 rte_atomic32_read(&cache_resource->refcnt));
1677                         rte_atomic32_inc(&cache_resource->refcnt);
1678                         dev_flow->dv.jump = cache_resource;
1679                         return 0;
1680                 }
1681         }
1682         /* Register new jump table resource. */
1683         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1684         if (!cache_resource)
1685                 return rte_flow_error_set(error, ENOMEM,
1686                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1687                                           "cannot allocate resource memory");
1688         *cache_resource = *resource;
1689         cache_resource->action =
1690                 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1691                 (resource->tbl->obj);
1692         if (!cache_resource->action) {
1693                 rte_free(cache_resource);
1694                 return rte_flow_error_set(error, ENOMEM,
1695                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1696                                           NULL, "cannot create action");
1697         }
1698         rte_atomic32_init(&cache_resource->refcnt);
1699         rte_atomic32_inc(&cache_resource->refcnt);
1700         LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1701         dev_flow->dv.jump = cache_resource;
1702         DRV_LOG(DEBUG, "new jump table  resource %p: refcnt %d++",
1703                 (void *)cache_resource,
1704                 rte_atomic32_read(&cache_resource->refcnt));
1705         return 0;
1706 }
1707
1708 /**
1709  * Find existing table port ID resource or create and register a new one.
1710  *
1711  * @param dev[in, out]
1712  *   Pointer to rte_eth_dev structure.
1713  * @param[in, out] resource
1714  *   Pointer to port ID action resource.
1715  * @parm[in, out] dev_flow
1716  *   Pointer to the dev_flow.
1717  * @param[out] error
1718  *   pointer to error structure.
1719  *
1720  * @return
1721  *   0 on success otherwise -errno and errno is set.
1722  */
1723 static int
1724 flow_dv_port_id_action_resource_register
1725                         (struct rte_eth_dev *dev,
1726                          struct mlx5_flow_dv_port_id_action_resource *resource,
1727                          struct mlx5_flow *dev_flow,
1728                          struct rte_flow_error *error)
1729 {
1730         struct mlx5_priv *priv = dev->data->dev_private;
1731         struct mlx5_ibv_shared *sh = priv->sh;
1732         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1733
1734         /* Lookup a matching resource from cache. */
1735         LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1736                 if (resource->port_id == cache_resource->port_id) {
1737                         DRV_LOG(DEBUG, "port id action resource resource %p: "
1738                                 "refcnt %d++",
1739                                 (void *)cache_resource,
1740                                 rte_atomic32_read(&cache_resource->refcnt));
1741                         rte_atomic32_inc(&cache_resource->refcnt);
1742                         dev_flow->dv.port_id_action = cache_resource;
1743                         return 0;
1744                 }
1745         }
1746         /* Register new port id action resource. */
1747         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1748         if (!cache_resource)
1749                 return rte_flow_error_set(error, ENOMEM,
1750                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1751                                           "cannot allocate resource memory");
1752         *cache_resource = *resource;
1753         cache_resource->action =
1754                 mlx5_glue->dr_create_flow_action_dest_vport
1755                         (priv->sh->fdb_domain, resource->port_id);
1756         if (!cache_resource->action) {
1757                 rte_free(cache_resource);
1758                 return rte_flow_error_set(error, ENOMEM,
1759                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1760                                           NULL, "cannot create action");
1761         }
1762         rte_atomic32_init(&cache_resource->refcnt);
1763         rte_atomic32_inc(&cache_resource->refcnt);
1764         LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1765         dev_flow->dv.port_id_action = cache_resource;
1766         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1767                 (void *)cache_resource,
1768                 rte_atomic32_read(&cache_resource->refcnt));
1769         return 0;
1770 }
1771
1772 /**
1773  * Find existing push vlan resource or create and register a new one.
1774  *
1775  * @param dev[in, out]
1776  *   Pointer to rte_eth_dev structure.
1777  * @param[in, out] resource
1778  *   Pointer to port ID action resource.
1779  * @parm[in, out] dev_flow
1780  *   Pointer to the dev_flow.
1781  * @param[out] error
1782  *   pointer to error structure.
1783  *
1784  * @return
1785  *   0 on success otherwise -errno and errno is set.
1786  */
1787 static int
1788 flow_dv_push_vlan_action_resource_register
1789                        (struct rte_eth_dev *dev,
1790                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
1791                         struct mlx5_flow *dev_flow,
1792                         struct rte_flow_error *error)
1793 {
1794         struct mlx5_priv *priv = dev->data->dev_private;
1795         struct mlx5_ibv_shared *sh = priv->sh;
1796         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
1797         struct mlx5dv_dr_domain *domain;
1798
1799         /* Lookup a matching resource from cache. */
1800         LIST_FOREACH(cache_resource, &sh->push_vlan_action_list, next) {
1801                 if (resource->vlan_tag == cache_resource->vlan_tag &&
1802                     resource->ft_type == cache_resource->ft_type) {
1803                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
1804                                 "refcnt %d++",
1805                                 (void *)cache_resource,
1806                                 rte_atomic32_read(&cache_resource->refcnt));
1807                         rte_atomic32_inc(&cache_resource->refcnt);
1808                         dev_flow->dv.push_vlan_res = cache_resource;
1809                         return 0;
1810                 }
1811         }
1812         /* Register new push_vlan action resource. */
1813         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1814         if (!cache_resource)
1815                 return rte_flow_error_set(error, ENOMEM,
1816                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1817                                           "cannot allocate resource memory");
1818         *cache_resource = *resource;
1819         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1820                 domain = sh->fdb_domain;
1821         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1822                 domain = sh->rx_domain;
1823         else
1824                 domain = sh->tx_domain;
1825         cache_resource->action =
1826                 mlx5_glue->dr_create_flow_action_push_vlan(domain,
1827                                                            resource->vlan_tag);
1828         if (!cache_resource->action) {
1829                 rte_free(cache_resource);
1830                 return rte_flow_error_set(error, ENOMEM,
1831                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1832                                           NULL, "cannot create action");
1833         }
1834         rte_atomic32_init(&cache_resource->refcnt);
1835         rte_atomic32_inc(&cache_resource->refcnt);
1836         LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next);
1837         dev_flow->dv.push_vlan_res = cache_resource;
1838         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
1839                 (void *)cache_resource,
1840                 rte_atomic32_read(&cache_resource->refcnt));
1841         return 0;
1842 }
1843 /**
1844  * Get the size of specific rte_flow_item_type
1845  *
1846  * @param[in] item_type
1847  *   Tested rte_flow_item_type.
1848  *
1849  * @return
1850  *   sizeof struct item_type, 0 if void or irrelevant.
1851  */
1852 static size_t
1853 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1854 {
1855         size_t retval;
1856
1857         switch (item_type) {
1858         case RTE_FLOW_ITEM_TYPE_ETH:
1859                 retval = sizeof(struct rte_flow_item_eth);
1860                 break;
1861         case RTE_FLOW_ITEM_TYPE_VLAN:
1862                 retval = sizeof(struct rte_flow_item_vlan);
1863                 break;
1864         case RTE_FLOW_ITEM_TYPE_IPV4:
1865                 retval = sizeof(struct rte_flow_item_ipv4);
1866                 break;
1867         case RTE_FLOW_ITEM_TYPE_IPV6:
1868                 retval = sizeof(struct rte_flow_item_ipv6);
1869                 break;
1870         case RTE_FLOW_ITEM_TYPE_UDP:
1871                 retval = sizeof(struct rte_flow_item_udp);
1872                 break;
1873         case RTE_FLOW_ITEM_TYPE_TCP:
1874                 retval = sizeof(struct rte_flow_item_tcp);
1875                 break;
1876         case RTE_FLOW_ITEM_TYPE_VXLAN:
1877                 retval = sizeof(struct rte_flow_item_vxlan);
1878                 break;
1879         case RTE_FLOW_ITEM_TYPE_GRE:
1880                 retval = sizeof(struct rte_flow_item_gre);
1881                 break;
1882         case RTE_FLOW_ITEM_TYPE_NVGRE:
1883                 retval = sizeof(struct rte_flow_item_nvgre);
1884                 break;
1885         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1886                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1887                 break;
1888         case RTE_FLOW_ITEM_TYPE_MPLS:
1889                 retval = sizeof(struct rte_flow_item_mpls);
1890                 break;
1891         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1892         default:
1893                 retval = 0;
1894                 break;
1895         }
1896         return retval;
1897 }
1898
1899 #define MLX5_ENCAP_IPV4_VERSION         0x40
1900 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
1901 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
1902 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
1903 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
1904 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
1905 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
1906
1907 /**
1908  * Convert the encap action data from list of rte_flow_item to raw buffer
1909  *
1910  * @param[in] items
1911  *   Pointer to rte_flow_item objects list.
1912  * @param[out] buf
1913  *   Pointer to the output buffer.
1914  * @param[out] size
1915  *   Pointer to the output buffer size.
1916  * @param[out] error
1917  *   Pointer to the error structure.
1918  *
1919  * @return
1920  *   0 on success, a negative errno value otherwise and rte_errno is set.
1921  */
1922 static int
1923 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1924                            size_t *size, struct rte_flow_error *error)
1925 {
1926         struct rte_ether_hdr *eth = NULL;
1927         struct rte_vlan_hdr *vlan = NULL;
1928         struct rte_ipv4_hdr *ipv4 = NULL;
1929         struct rte_ipv6_hdr *ipv6 = NULL;
1930         struct rte_udp_hdr *udp = NULL;
1931         struct rte_vxlan_hdr *vxlan = NULL;
1932         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1933         struct rte_gre_hdr *gre = NULL;
1934         size_t len;
1935         size_t temp_size = 0;
1936
1937         if (!items)
1938                 return rte_flow_error_set(error, EINVAL,
1939                                           RTE_FLOW_ERROR_TYPE_ACTION,
1940                                           NULL, "invalid empty data");
1941         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1942                 len = flow_dv_get_item_len(items->type);
1943                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1944                         return rte_flow_error_set(error, EINVAL,
1945                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1946                                                   (void *)items->type,
1947                                                   "items total size is too big"
1948                                                   " for encap action");
1949                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1950                 switch (items->type) {
1951                 case RTE_FLOW_ITEM_TYPE_ETH:
1952                         eth = (struct rte_ether_hdr *)&buf[temp_size];
1953                         break;
1954                 case RTE_FLOW_ITEM_TYPE_VLAN:
1955                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1956                         if (!eth)
1957                                 return rte_flow_error_set(error, EINVAL,
1958                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1959                                                 (void *)items->type,
1960                                                 "eth header not found");
1961                         if (!eth->ether_type)
1962                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1963                         break;
1964                 case RTE_FLOW_ITEM_TYPE_IPV4:
1965                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1966                         if (!vlan && !eth)
1967                                 return rte_flow_error_set(error, EINVAL,
1968                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1969                                                 (void *)items->type,
1970                                                 "neither eth nor vlan"
1971                                                 " header found");
1972                         if (vlan && !vlan->eth_proto)
1973                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1974                         else if (eth && !eth->ether_type)
1975                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1976                         if (!ipv4->version_ihl)
1977                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1978                                                     MLX5_ENCAP_IPV4_IHL_MIN;
1979                         if (!ipv4->time_to_live)
1980                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1981                         break;
1982                 case RTE_FLOW_ITEM_TYPE_IPV6:
1983                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1984                         if (!vlan && !eth)
1985                                 return rte_flow_error_set(error, EINVAL,
1986                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1987                                                 (void *)items->type,
1988                                                 "neither eth nor vlan"
1989                                                 " header found");
1990                         if (vlan && !vlan->eth_proto)
1991                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1992                         else if (eth && !eth->ether_type)
1993                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1994                         if (!ipv6->vtc_flow)
1995                                 ipv6->vtc_flow =
1996                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1997                         if (!ipv6->hop_limits)
1998                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1999                         break;
2000                 case RTE_FLOW_ITEM_TYPE_UDP:
2001                         udp = (struct rte_udp_hdr *)&buf[temp_size];
2002                         if (!ipv4 && !ipv6)
2003                                 return rte_flow_error_set(error, EINVAL,
2004                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2005                                                 (void *)items->type,
2006                                                 "ip header not found");
2007                         if (ipv4 && !ipv4->next_proto_id)
2008                                 ipv4->next_proto_id = IPPROTO_UDP;
2009                         else if (ipv6 && !ipv6->proto)
2010                                 ipv6->proto = IPPROTO_UDP;
2011                         break;
2012                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2013                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
2014                         if (!udp)
2015                                 return rte_flow_error_set(error, EINVAL,
2016                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2017                                                 (void *)items->type,
2018                                                 "udp header not found");
2019                         if (!udp->dst_port)
2020                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
2021                         if (!vxlan->vx_flags)
2022                                 vxlan->vx_flags =
2023                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
2024                         break;
2025                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2026                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
2027                         if (!udp)
2028                                 return rte_flow_error_set(error, EINVAL,
2029                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2030                                                 (void *)items->type,
2031                                                 "udp header not found");
2032                         if (!vxlan_gpe->proto)
2033                                 return rte_flow_error_set(error, EINVAL,
2034                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2035                                                 (void *)items->type,
2036                                                 "next protocol not found");
2037                         if (!udp->dst_port)
2038                                 udp->dst_port =
2039                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
2040                         if (!vxlan_gpe->vx_flags)
2041                                 vxlan_gpe->vx_flags =
2042                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
2043                         break;
2044                 case RTE_FLOW_ITEM_TYPE_GRE:
2045                 case RTE_FLOW_ITEM_TYPE_NVGRE:
2046                         gre = (struct rte_gre_hdr *)&buf[temp_size];
2047                         if (!gre->proto)
2048                                 return rte_flow_error_set(error, EINVAL,
2049                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2050                                                 (void *)items->type,
2051                                                 "next protocol not found");
2052                         if (!ipv4 && !ipv6)
2053                                 return rte_flow_error_set(error, EINVAL,
2054                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2055                                                 (void *)items->type,
2056                                                 "ip header not found");
2057                         if (ipv4 && !ipv4->next_proto_id)
2058                                 ipv4->next_proto_id = IPPROTO_GRE;
2059                         else if (ipv6 && !ipv6->proto)
2060                                 ipv6->proto = IPPROTO_GRE;
2061                         break;
2062                 case RTE_FLOW_ITEM_TYPE_VOID:
2063                         break;
2064                 default:
2065                         return rte_flow_error_set(error, EINVAL,
2066                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2067                                                   (void *)items->type,
2068                                                   "unsupported item type");
2069                         break;
2070                 }
2071                 temp_size += len;
2072         }
2073         *size = temp_size;
2074         return 0;
2075 }
2076
2077 static int
2078 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
2079 {
2080         struct rte_ether_hdr *eth = NULL;
2081         struct rte_vlan_hdr *vlan = NULL;
2082         struct rte_ipv6_hdr *ipv6 = NULL;
2083         struct rte_udp_hdr *udp = NULL;
2084         char *next_hdr;
2085         uint16_t proto;
2086
2087         eth = (struct rte_ether_hdr *)data;
2088         next_hdr = (char *)(eth + 1);
2089         proto = RTE_BE16(eth->ether_type);
2090
2091         /* VLAN skipping */
2092         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
2093                 vlan = (struct rte_vlan_hdr *)next_hdr;
2094                 proto = RTE_BE16(vlan->eth_proto);
2095                 next_hdr += sizeof(struct rte_vlan_hdr);
2096         }
2097
2098         /* HW calculates IPv4 csum. no need to proceed */
2099         if (proto == RTE_ETHER_TYPE_IPV4)
2100                 return 0;
2101
2102         /* non IPv4/IPv6 header. not supported */
2103         if (proto != RTE_ETHER_TYPE_IPV6) {
2104                 return rte_flow_error_set(error, ENOTSUP,
2105                                           RTE_FLOW_ERROR_TYPE_ACTION,
2106                                           NULL, "Cannot offload non IPv4/IPv6");
2107         }
2108
2109         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
2110
2111         /* ignore non UDP */
2112         if (ipv6->proto != IPPROTO_UDP)
2113                 return 0;
2114
2115         udp = (struct rte_udp_hdr *)(ipv6 + 1);
2116         udp->dgram_cksum = 0;
2117
2118         return 0;
2119 }
2120
2121 /**
2122  * Convert L2 encap action to DV specification.
2123  *
2124  * @param[in] dev
2125  *   Pointer to rte_eth_dev structure.
2126  * @param[in] action
2127  *   Pointer to action structure.
2128  * @param[in, out] dev_flow
2129  *   Pointer to the mlx5_flow.
2130  * @param[in] transfer
2131  *   Mark if the flow is E-Switch flow.
2132  * @param[out] error
2133  *   Pointer to the error structure.
2134  *
2135  * @return
2136  *   0 on success, a negative errno value otherwise and rte_errno is set.
2137  */
2138 static int
2139 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
2140                                const struct rte_flow_action *action,
2141                                struct mlx5_flow *dev_flow,
2142                                uint8_t transfer,
2143                                struct rte_flow_error *error)
2144 {
2145         const struct rte_flow_item *encap_data;
2146         const struct rte_flow_action_raw_encap *raw_encap_data;
2147         struct mlx5_flow_dv_encap_decap_resource res = {
2148                 .reformat_type =
2149                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
2150                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
2151                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
2152         };
2153
2154         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
2155                 raw_encap_data =
2156                         (const struct rte_flow_action_raw_encap *)action->conf;
2157                 res.size = raw_encap_data->size;
2158                 memcpy(res.buf, raw_encap_data->data, res.size);
2159                 if (flow_dv_zero_encap_udp_csum(res.buf, error))
2160                         return -rte_errno;
2161         } else {
2162                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
2163                         encap_data =
2164                                 ((const struct rte_flow_action_vxlan_encap *)
2165                                                 action->conf)->definition;
2166                 else
2167                         encap_data =
2168                                 ((const struct rte_flow_action_nvgre_encap *)
2169                                                 action->conf)->definition;
2170                 if (flow_dv_convert_encap_data(encap_data, res.buf,
2171                                                &res.size, error))
2172                         return -rte_errno;
2173         }
2174         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
2175                 return rte_flow_error_set(error, EINVAL,
2176                                           RTE_FLOW_ERROR_TYPE_ACTION,
2177                                           NULL, "can't create L2 encap action");
2178         return 0;
2179 }
2180
2181 /**
2182  * Convert L2 decap action to DV specification.
2183  *
2184  * @param[in] dev
2185  *   Pointer to rte_eth_dev structure.
2186  * @param[in, out] dev_flow
2187  *   Pointer to the mlx5_flow.
2188  * @param[in] transfer
2189  *   Mark if the flow is E-Switch flow.
2190  * @param[out] error
2191  *   Pointer to the error structure.
2192  *
2193  * @return
2194  *   0 on success, a negative errno value otherwise and rte_errno is set.
2195  */
2196 static int
2197 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
2198                                struct mlx5_flow *dev_flow,
2199                                uint8_t transfer,
2200                                struct rte_flow_error *error)
2201 {
2202         struct mlx5_flow_dv_encap_decap_resource res = {
2203                 .size = 0,
2204                 .reformat_type =
2205                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
2206                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
2207                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
2208         };
2209
2210         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
2211                 return rte_flow_error_set(error, EINVAL,
2212                                           RTE_FLOW_ERROR_TYPE_ACTION,
2213                                           NULL, "can't create L2 decap action");
2214         return 0;
2215 }
2216
2217 /**
2218  * Convert raw decap/encap (L3 tunnel) action to DV specification.
2219  *
2220  * @param[in] dev
2221  *   Pointer to rte_eth_dev structure.
2222  * @param[in] action
2223  *   Pointer to action structure.
2224  * @param[in, out] dev_flow
2225  *   Pointer to the mlx5_flow.
2226  * @param[in] attr
2227  *   Pointer to the flow attributes.
2228  * @param[out] error
2229  *   Pointer to the error structure.
2230  *
2231  * @return
2232  *   0 on success, a negative errno value otherwise and rte_errno is set.
2233  */
2234 static int
2235 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
2236                                 const struct rte_flow_action *action,
2237                                 struct mlx5_flow *dev_flow,
2238                                 const struct rte_flow_attr *attr,
2239                                 struct rte_flow_error *error)
2240 {
2241         const struct rte_flow_action_raw_encap *encap_data;
2242         struct mlx5_flow_dv_encap_decap_resource res;
2243
2244         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
2245         res.size = encap_data->size;
2246         memcpy(res.buf, encap_data->data, res.size);
2247         res.reformat_type = attr->egress ?
2248                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
2249                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
2250         if (attr->transfer)
2251                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
2252         else
2253                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
2254                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
2255         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
2256                 return rte_flow_error_set(error, EINVAL,
2257                                           RTE_FLOW_ERROR_TYPE_ACTION,
2258                                           NULL, "can't create encap action");
2259         return 0;
2260 }
2261
2262 /**
2263  * Create action push VLAN.
2264  *
2265  * @param[in] dev
2266  *   Pointer to rte_eth_dev structure.
2267  * @param[in] vlan_tag
2268  *   the vlan tag to push to the Ethernet header.
2269  * @param[in, out] dev_flow
2270  *   Pointer to the mlx5_flow.
2271  * @param[in] attr
2272  *   Pointer to the flow attributes.
2273  * @param[out] error
2274  *   Pointer to the error structure.
2275  *
2276  * @return
2277  *   0 on success, a negative errno value otherwise and rte_errno is set.
2278  */
2279 static int
2280 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
2281                                 const struct rte_flow_attr *attr,
2282                                 const struct rte_vlan_hdr *vlan,
2283                                 struct mlx5_flow *dev_flow,
2284                                 struct rte_flow_error *error)
2285 {
2286         struct mlx5_flow_dv_push_vlan_action_resource res;
2287
2288         res.vlan_tag =
2289                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
2290                                  vlan->vlan_tci);
2291         if (attr->transfer)
2292                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
2293         else
2294                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
2295                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
2296         return flow_dv_push_vlan_action_resource_register
2297                                             (dev, &res, dev_flow, error);
2298 }
2299
2300 /**
2301  * Validate the modify-header actions.
2302  *
2303  * @param[in] action_flags
2304  *   Holds the actions detected until now.
2305  * @param[in] action
2306  *   Pointer to the modify action.
2307  * @param[out] error
2308  *   Pointer to error structure.
2309  *
2310  * @return
2311  *   0 on success, a negative errno value otherwise and rte_errno is set.
2312  */
2313 static int
2314 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
2315                                    const struct rte_flow_action *action,
2316                                    struct rte_flow_error *error)
2317 {
2318         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2321                                           NULL, "action configuration not set");
2322         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
2323                 return rte_flow_error_set(error, EINVAL,
2324                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2325                                           "can't have encap action before"
2326                                           " modify action");
2327         return 0;
2328 }
2329
2330 /**
2331  * Validate the modify-header MAC address actions.
2332  *
2333  * @param[in] action_flags
2334  *   Holds the actions detected until now.
2335  * @param[in] action
2336  *   Pointer to the modify action.
2337  * @param[in] item_flags
2338  *   Holds the items detected.
2339  * @param[out] error
2340  *   Pointer to error structure.
2341  *
2342  * @return
2343  *   0 on success, a negative errno value otherwise and rte_errno is set.
2344  */
2345 static int
2346 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
2347                                    const struct rte_flow_action *action,
2348                                    const uint64_t item_flags,
2349                                    struct rte_flow_error *error)
2350 {
2351         int ret = 0;
2352
2353         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2354         if (!ret) {
2355                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
2356                         return rte_flow_error_set(error, EINVAL,
2357                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2358                                                   NULL,
2359                                                   "no L2 item in pattern");
2360         }
2361         return ret;
2362 }
2363
2364 /**
2365  * Validate the modify-header IPv4 address actions.
2366  *
2367  * @param[in] action_flags
2368  *   Holds the actions detected until now.
2369  * @param[in] action
2370  *   Pointer to the modify action.
2371  * @param[in] item_flags
2372  *   Holds the items detected.
2373  * @param[out] error
2374  *   Pointer to error structure.
2375  *
2376  * @return
2377  *   0 on success, a negative errno value otherwise and rte_errno is set.
2378  */
2379 static int
2380 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
2381                                     const struct rte_flow_action *action,
2382                                     const uint64_t item_flags,
2383                                     struct rte_flow_error *error)
2384 {
2385         int ret = 0;
2386
2387         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2388         if (!ret) {
2389                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
2390                         return rte_flow_error_set(error, EINVAL,
2391                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2392                                                   NULL,
2393                                                   "no ipv4 item in pattern");
2394         }
2395         return ret;
2396 }
2397
2398 /**
2399  * Validate the modify-header IPv6 address actions.
2400  *
2401  * @param[in] action_flags
2402  *   Holds the actions detected until now.
2403  * @param[in] action
2404  *   Pointer to the modify action.
2405  * @param[in] item_flags
2406  *   Holds the items detected.
2407  * @param[out] error
2408  *   Pointer to error structure.
2409  *
2410  * @return
2411  *   0 on success, a negative errno value otherwise and rte_errno is set.
2412  */
2413 static int
2414 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
2415                                     const struct rte_flow_action *action,
2416                                     const uint64_t item_flags,
2417                                     struct rte_flow_error *error)
2418 {
2419         int ret = 0;
2420
2421         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2422         if (!ret) {
2423                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
2424                         return rte_flow_error_set(error, EINVAL,
2425                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2426                                                   NULL,
2427                                                   "no ipv6 item in pattern");
2428         }
2429         return ret;
2430 }
2431
2432 /**
2433  * Validate the modify-header TP actions.
2434  *
2435  * @param[in] action_flags
2436  *   Holds the actions detected until now.
2437  * @param[in] action
2438  *   Pointer to the modify action.
2439  * @param[in] item_flags
2440  *   Holds the items detected.
2441  * @param[out] error
2442  *   Pointer to error structure.
2443  *
2444  * @return
2445  *   0 on success, a negative errno value otherwise and rte_errno is set.
2446  */
2447 static int
2448 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
2449                                   const struct rte_flow_action *action,
2450                                   const uint64_t item_flags,
2451                                   struct rte_flow_error *error)
2452 {
2453         int ret = 0;
2454
2455         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2456         if (!ret) {
2457                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
2458                         return rte_flow_error_set(error, EINVAL,
2459                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2460                                                   NULL, "no transport layer "
2461                                                   "in pattern");
2462         }
2463         return ret;
2464 }
2465
2466 /**
2467  * Validate the modify-header actions of increment/decrement
2468  * TCP Sequence-number.
2469  *
2470  * @param[in] action_flags
2471  *   Holds the actions detected until now.
2472  * @param[in] action
2473  *   Pointer to the modify action.
2474  * @param[in] item_flags
2475  *   Holds the items detected.
2476  * @param[out] error
2477  *   Pointer to error structure.
2478  *
2479  * @return
2480  *   0 on success, a negative errno value otherwise and rte_errno is set.
2481  */
2482 static int
2483 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
2484                                        const struct rte_flow_action *action,
2485                                        const uint64_t item_flags,
2486                                        struct rte_flow_error *error)
2487 {
2488         int ret = 0;
2489
2490         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2491         if (!ret) {
2492                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2493                         return rte_flow_error_set(error, EINVAL,
2494                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2495                                                   NULL, "no TCP item in"
2496                                                   " pattern");
2497                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
2498                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
2499                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
2500                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
2501                         return rte_flow_error_set(error, EINVAL,
2502                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2503                                                   NULL,
2504                                                   "cannot decrease and increase"
2505                                                   " TCP sequence number"
2506                                                   " at the same time");
2507         }
2508         return ret;
2509 }
2510
2511 /**
2512  * Validate the modify-header actions of increment/decrement
2513  * TCP Acknowledgment number.
2514  *
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] action
2518  *   Pointer to the modify action.
2519  * @param[in] item_flags
2520  *   Holds the items detected.
2521  * @param[out] error
2522  *   Pointer to error structure.
2523  *
2524  * @return
2525  *   0 on success, a negative errno value otherwise and rte_errno is set.
2526  */
2527 static int
2528 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
2529                                        const struct rte_flow_action *action,
2530                                        const uint64_t item_flags,
2531                                        struct rte_flow_error *error)
2532 {
2533         int ret = 0;
2534
2535         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2536         if (!ret) {
2537                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2538                         return rte_flow_error_set(error, EINVAL,
2539                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2540                                                   NULL, "no TCP item in"
2541                                                   " pattern");
2542                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
2543                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
2544                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
2545                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
2546                         return rte_flow_error_set(error, EINVAL,
2547                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2548                                                   NULL,
2549                                                   "cannot decrease and increase"
2550                                                   " TCP acknowledgment number"
2551                                                   " at the same time");
2552         }
2553         return ret;
2554 }
2555
2556 /**
2557  * Validate the modify-header TTL actions.
2558  *
2559  * @param[in] action_flags
2560  *   Holds the actions detected until now.
2561  * @param[in] action
2562  *   Pointer to the modify action.
2563  * @param[in] item_flags
2564  *   Holds the items detected.
2565  * @param[out] error
2566  *   Pointer to error structure.
2567  *
2568  * @return
2569  *   0 on success, a negative errno value otherwise and rte_errno is set.
2570  */
2571 static int
2572 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
2573                                    const struct rte_flow_action *action,
2574                                    const uint64_t item_flags,
2575                                    struct rte_flow_error *error)
2576 {
2577         int ret = 0;
2578
2579         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2580         if (!ret) {
2581                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
2582                         return rte_flow_error_set(error, EINVAL,
2583                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2584                                                   NULL,
2585                                                   "no IP protocol in pattern");
2586         }
2587         return ret;
2588 }
2589
2590 /**
2591  * Validate jump action.
2592  *
2593  * @param[in] action
2594  *   Pointer to the jump action.
2595  * @param[in] action_flags
2596  *   Holds the actions detected until now.
2597  * @param[in] attributes
2598  *   Pointer to flow attributes
2599  * @param[in] external
2600  *   Action belongs to flow rule created by request external to PMD.
2601  * @param[out] error
2602  *   Pointer to error structure.
2603  *
2604  * @return
2605  *   0 on success, a negative errno value otherwise and rte_errno is set.
2606  */
2607 static int
2608 flow_dv_validate_action_jump(const struct rte_flow_action *action,
2609                              uint64_t action_flags,
2610                              const struct rte_flow_attr *attributes,
2611                              bool external, struct rte_flow_error *error)
2612 {
2613         uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB :
2614                                                     MLX5_MAX_TABLES;
2615         uint32_t target_group, table;
2616         int ret = 0;
2617
2618         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2619                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2620                 return rte_flow_error_set(error, EINVAL,
2621                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2622                                           "can't have 2 fate actions in"
2623                                           " same flow");
2624         if (!action->conf)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2627                                           NULL, "action configuration not set");
2628         target_group =
2629                 ((const struct rte_flow_action_jump *)action->conf)->group;
2630         ret = mlx5_flow_group_to_table(attributes, external, target_group,
2631                                        &table, error);
2632         if (ret)
2633                 return ret;
2634         if (table >= max_group)
2635                 return rte_flow_error_set(error, EINVAL,
2636                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
2637                                           "target group index out of range");
2638         if (attributes->group >= target_group)
2639                 return rte_flow_error_set(error, EINVAL,
2640                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2641                                           "target group must be higher than"
2642                                           " the current flow group");
2643         return 0;
2644 }
2645
2646 /*
2647  * Validate the port_id action.
2648  *
2649  * @param[in] dev
2650  *   Pointer to rte_eth_dev structure.
2651  * @param[in] action_flags
2652  *   Bit-fields that holds the actions detected until now.
2653  * @param[in] action
2654  *   Port_id RTE action structure.
2655  * @param[in] attr
2656  *   Attributes of flow that includes this action.
2657  * @param[out] error
2658  *   Pointer to error structure.
2659  *
2660  * @return
2661  *   0 on success, a negative errno value otherwise and rte_errno is set.
2662  */
2663 static int
2664 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2665                                 uint64_t action_flags,
2666                                 const struct rte_flow_action *action,
2667                                 const struct rte_flow_attr *attr,
2668                                 struct rte_flow_error *error)
2669 {
2670         const struct rte_flow_action_port_id *port_id;
2671         struct mlx5_priv *act_priv;
2672         struct mlx5_priv *dev_priv;
2673         uint16_t port;
2674
2675         if (!attr->transfer)
2676                 return rte_flow_error_set(error, ENOTSUP,
2677                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2678                                           NULL,
2679                                           "port id action is valid in transfer"
2680                                           " mode only");
2681         if (!action || !action->conf)
2682                 return rte_flow_error_set(error, ENOTSUP,
2683                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2684                                           NULL,
2685                                           "port id action parameters must be"
2686                                           " specified");
2687         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2688                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2689                 return rte_flow_error_set(error, EINVAL,
2690                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2691                                           "can have only one fate actions in"
2692                                           " a flow");
2693         dev_priv = mlx5_dev_to_eswitch_info(dev);
2694         if (!dev_priv)
2695                 return rte_flow_error_set(error, rte_errno,
2696                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2697                                           NULL,
2698                                           "failed to obtain E-Switch info");
2699         port_id = action->conf;
2700         port = port_id->original ? dev->data->port_id : port_id->id;
2701         act_priv = mlx5_port_to_eswitch_info(port, false);
2702         if (!act_priv)
2703                 return rte_flow_error_set
2704                                 (error, rte_errno,
2705                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2706                                  "failed to obtain E-Switch port id for port");
2707         if (act_priv->domain_id != dev_priv->domain_id)
2708                 return rte_flow_error_set
2709                                 (error, EINVAL,
2710                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2711                                  "port does not belong to"
2712                                  " E-Switch being configured");
2713         return 0;
2714 }
2715
2716 /**
2717  * Find existing modify-header resource or create and register a new one.
2718  *
2719  * @param dev[in, out]
2720  *   Pointer to rte_eth_dev structure.
2721  * @param[in, out] resource
2722  *   Pointer to modify-header resource.
2723  * @parm[in, out] dev_flow
2724  *   Pointer to the dev_flow.
2725  * @param[out] error
2726  *   pointer to error structure.
2727  *
2728  * @return
2729  *   0 on success otherwise -errno and errno is set.
2730  */
2731 static int
2732 flow_dv_modify_hdr_resource_register
2733                         (struct rte_eth_dev *dev,
2734                          struct mlx5_flow_dv_modify_hdr_resource *resource,
2735                          struct mlx5_flow *dev_flow,
2736                          struct rte_flow_error *error)
2737 {
2738         struct mlx5_priv *priv = dev->data->dev_private;
2739         struct mlx5_ibv_shared *sh = priv->sh;
2740         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2741         struct mlx5dv_dr_domain *ns;
2742
2743         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2744                 ns = sh->fdb_domain;
2745         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2746                 ns = sh->tx_domain;
2747         else
2748                 ns = sh->rx_domain;
2749         resource->flags =
2750                 dev_flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2751         /* Lookup a matching resource from cache. */
2752         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2753                 if (resource->ft_type == cache_resource->ft_type &&
2754                     resource->actions_num == cache_resource->actions_num &&
2755                     resource->flags == cache_resource->flags &&
2756                     !memcmp((const void *)resource->actions,
2757                             (const void *)cache_resource->actions,
2758                             (resource->actions_num *
2759                                             sizeof(resource->actions[0])))) {
2760                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2761                                 (void *)cache_resource,
2762                                 rte_atomic32_read(&cache_resource->refcnt));
2763                         rte_atomic32_inc(&cache_resource->refcnt);
2764                         dev_flow->dv.modify_hdr = cache_resource;
2765                         return 0;
2766                 }
2767         }
2768         /* Register new modify-header resource. */
2769         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2770         if (!cache_resource)
2771                 return rte_flow_error_set(error, ENOMEM,
2772                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2773                                           "cannot allocate resource memory");
2774         *cache_resource = *resource;
2775         cache_resource->verbs_action =
2776                 mlx5_glue->dv_create_flow_action_modify_header
2777                                         (sh->ctx, cache_resource->ft_type,
2778                                          ns, cache_resource->flags,
2779                                          cache_resource->actions_num *
2780                                          sizeof(cache_resource->actions[0]),
2781                                          (uint64_t *)cache_resource->actions);
2782         if (!cache_resource->verbs_action) {
2783                 rte_free(cache_resource);
2784                 return rte_flow_error_set(error, ENOMEM,
2785                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2786                                           NULL, "cannot create action");
2787         }
2788         rte_atomic32_init(&cache_resource->refcnt);
2789         rte_atomic32_inc(&cache_resource->refcnt);
2790         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2791         dev_flow->dv.modify_hdr = cache_resource;
2792         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2793                 (void *)cache_resource,
2794                 rte_atomic32_read(&cache_resource->refcnt));
2795         return 0;
2796 }
2797
2798 #define MLX5_CNT_CONTAINER_RESIZE 64
2799
2800 /**
2801  * Get or create a flow counter.
2802  *
2803  * @param[in] dev
2804  *   Pointer to the Ethernet device structure.
2805  * @param[in] shared
2806  *   Indicate if this counter is shared with other flows.
2807  * @param[in] id
2808  *   Counter identifier.
2809  *
2810  * @return
2811  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2812  */
2813 static struct mlx5_flow_counter *
2814 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2815                                uint32_t id)
2816 {
2817         struct mlx5_priv *priv = dev->data->dev_private;
2818         struct mlx5_flow_counter *cnt = NULL;
2819         struct mlx5_devx_obj *dcs = NULL;
2820
2821         if (!priv->config.devx) {
2822                 rte_errno = ENOTSUP;
2823                 return NULL;
2824         }
2825         if (shared) {
2826                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2827                         if (cnt->shared && cnt->id == id) {
2828                                 cnt->ref_cnt++;
2829                                 return cnt;
2830                         }
2831                 }
2832         }
2833         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2834         if (!dcs)
2835                 return NULL;
2836         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2837         if (!cnt) {
2838                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2839                 rte_errno = ENOMEM;
2840                 return NULL;
2841         }
2842         struct mlx5_flow_counter tmpl = {
2843                 .shared = shared,
2844                 .ref_cnt = 1,
2845                 .id = id,
2846                 .dcs = dcs,
2847         };
2848         tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2849         if (!tmpl.action) {
2850                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2851                 rte_errno = errno;
2852                 rte_free(cnt);
2853                 return NULL;
2854         }
2855         *cnt = tmpl;
2856         TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2857         return cnt;
2858 }
2859
2860 /**
2861  * Release a flow counter.
2862  *
2863  * @param[in] dev
2864  *   Pointer to the Ethernet device structure.
2865  * @param[in] counter
2866  *   Pointer to the counter handler.
2867  */
2868 static void
2869 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2870                                  struct mlx5_flow_counter *counter)
2871 {
2872         struct mlx5_priv *priv = dev->data->dev_private;
2873
2874         if (!counter)
2875                 return;
2876         if (--counter->ref_cnt == 0) {
2877                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2878                 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2879                 rte_free(counter);
2880         }
2881 }
2882
2883 /**
2884  * Query a devx flow counter.
2885  *
2886  * @param[in] dev
2887  *   Pointer to the Ethernet device structure.
2888  * @param[in] cnt
2889  *   Pointer to the flow counter.
2890  * @param[out] pkts
2891  *   The statistics value of packets.
2892  * @param[out] bytes
2893  *   The statistics value of bytes.
2894  *
2895  * @return
2896  *   0 on success, otherwise a negative errno value and rte_errno is set.
2897  */
2898 static inline int
2899 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2900                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2901                      uint64_t *bytes)
2902 {
2903         return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2904                                                 0, NULL, NULL, 0);
2905 }
2906
2907 /**
2908  * Get a pool by a counter.
2909  *
2910  * @param[in] cnt
2911  *   Pointer to the counter.
2912  *
2913  * @return
2914  *   The counter pool.
2915  */
2916 static struct mlx5_flow_counter_pool *
2917 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2918 {
2919         if (!cnt->batch) {
2920                 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2921                 return (struct mlx5_flow_counter_pool *)cnt - 1;
2922         }
2923         return cnt->pool;
2924 }
2925
2926 /**
2927  * Get a pool by devx counter ID.
2928  *
2929  * @param[in] cont
2930  *   Pointer to the counter container.
2931  * @param[in] id
2932  *   The counter devx ID.
2933  *
2934  * @return
2935  *   The counter pool pointer if exists, NULL otherwise,
2936  */
2937 static struct mlx5_flow_counter_pool *
2938 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2939 {
2940         struct mlx5_flow_counter_pool *pool;
2941
2942         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2943                 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2944                                 MLX5_COUNTERS_PER_POOL;
2945
2946                 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2947                         return pool;
2948         };
2949         return NULL;
2950 }
2951
2952 /**
2953  * Allocate a new memory for the counter values wrapped by all the needed
2954  * management.
2955  *
2956  * @param[in] dev
2957  *   Pointer to the Ethernet device structure.
2958  * @param[in] raws_n
2959  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2960  *
2961  * @return
2962  *   The new memory management pointer on success, otherwise NULL and rte_errno
2963  *   is set.
2964  */
2965 static struct mlx5_counter_stats_mem_mng *
2966 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2967 {
2968         struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2969                                         (dev->data->dev_private))->sh;
2970         struct mlx5_devx_mkey_attr mkey_attr;
2971         struct mlx5_counter_stats_mem_mng *mem_mng;
2972         volatile struct flow_counter_stats *raw_data;
2973         int size = (sizeof(struct flow_counter_stats) *
2974                         MLX5_COUNTERS_PER_POOL +
2975                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2976                         sizeof(struct mlx5_counter_stats_mem_mng);
2977         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2978         int i;
2979
2980         if (!mem) {
2981                 rte_errno = ENOMEM;
2982                 return NULL;
2983         }
2984         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2985         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2986         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2987                                                  IBV_ACCESS_LOCAL_WRITE);
2988         if (!mem_mng->umem) {
2989                 rte_errno = errno;
2990                 rte_free(mem);
2991                 return NULL;
2992         }
2993         mkey_attr.addr = (uintptr_t)mem;
2994         mkey_attr.size = size;
2995         mkey_attr.umem_id = mem_mng->umem->umem_id;
2996         mkey_attr.pd = sh->pdn;
2997         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2998         if (!mem_mng->dm) {
2999                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
3000                 rte_errno = errno;
3001                 rte_free(mem);
3002                 return NULL;
3003         }
3004         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
3005         raw_data = (volatile struct flow_counter_stats *)mem;
3006         for (i = 0; i < raws_n; ++i) {
3007                 mem_mng->raws[i].mem_mng = mem_mng;
3008                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
3009         }
3010         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
3011         return mem_mng;
3012 }
3013
3014 /**
3015  * Resize a counter container.
3016  *
3017  * @param[in] dev
3018  *   Pointer to the Ethernet device structure.
3019  * @param[in] batch
3020  *   Whether the pool is for counter that was allocated by batch command.
3021  *
3022  * @return
3023  *   The new container pointer on success, otherwise NULL and rte_errno is set.
3024  */
3025 static struct mlx5_pools_container *
3026 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
3027 {
3028         struct mlx5_priv *priv = dev->data->dev_private;
3029         struct mlx5_pools_container *cont =
3030                         MLX5_CNT_CONTAINER(priv->sh, batch, 0);
3031         struct mlx5_pools_container *new_cont =
3032                         MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
3033         struct mlx5_counter_stats_mem_mng *mem_mng;
3034         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
3035         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
3036         int i;
3037
3038         if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
3039                 /* The last resize still hasn't detected by the host thread. */
3040                 rte_errno = EAGAIN;
3041                 return NULL;
3042         }
3043         new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
3044         if (!new_cont->pools) {
3045                 rte_errno = ENOMEM;
3046                 return NULL;
3047         }
3048         if (cont->n)
3049                 memcpy(new_cont->pools, cont->pools, cont->n *
3050                        sizeof(struct mlx5_flow_counter_pool *));
3051         mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
3052                 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
3053         if (!mem_mng) {
3054                 rte_free(new_cont->pools);
3055                 return NULL;
3056         }
3057         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
3058                 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
3059                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
3060                                  i, next);
3061         new_cont->n = resize;
3062         rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
3063         TAILQ_INIT(&new_cont->pool_list);
3064         TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
3065         new_cont->init_mem_mng = mem_mng;
3066         rte_cio_wmb();
3067          /* Flip the master container. */
3068         priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
3069         return new_cont;
3070 }
3071
3072 /**
3073  * Query a devx flow counter.
3074  *
3075  * @param[in] dev
3076  *   Pointer to the Ethernet device structure.
3077  * @param[in] cnt
3078  *   Pointer to the flow counter.
3079  * @param[out] pkts
3080  *   The statistics value of packets.
3081  * @param[out] bytes
3082  *   The statistics value of bytes.
3083  *
3084  * @return
3085  *   0 on success, otherwise a negative errno value and rte_errno is set.
3086  */
3087 static inline int
3088 _flow_dv_query_count(struct rte_eth_dev *dev,
3089                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
3090                      uint64_t *bytes)
3091 {
3092         struct mlx5_priv *priv = dev->data->dev_private;
3093         struct mlx5_flow_counter_pool *pool =
3094                         flow_dv_counter_pool_get(cnt);
3095         int offset = cnt - &pool->counters_raw[0];
3096
3097         if (priv->counter_fallback)
3098                 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
3099
3100         rte_spinlock_lock(&pool->sl);
3101         /*
3102          * The single counters allocation may allocate smaller ID than the
3103          * current allocated in parallel to the host reading.
3104          * In this case the new counter values must be reported as 0.
3105          */
3106         if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
3107                 *pkts = 0;
3108                 *bytes = 0;
3109         } else {
3110                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
3111                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
3112         }
3113         rte_spinlock_unlock(&pool->sl);
3114         return 0;
3115 }
3116
3117 /**
3118  * Create and initialize a new counter pool.
3119  *
3120  * @param[in] dev
3121  *   Pointer to the Ethernet device structure.
3122  * @param[out] dcs
3123  *   The devX counter handle.
3124  * @param[in] batch
3125  *   Whether the pool is for counter that was allocated by batch command.
3126  *
3127  * @return
3128  *   A new pool pointer on success, NULL otherwise and rte_errno is set.
3129  */
3130 static struct mlx5_flow_counter_pool *
3131 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
3132                     uint32_t batch)
3133 {
3134         struct mlx5_priv *priv = dev->data->dev_private;
3135         struct mlx5_flow_counter_pool *pool;
3136         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
3137                                                                0);
3138         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
3139         uint32_t size;
3140
3141         if (cont->n == n_valid) {
3142                 cont = flow_dv_container_resize(dev, batch);
3143                 if (!cont)
3144                         return NULL;
3145         }
3146         size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
3147                         sizeof(struct mlx5_flow_counter);
3148         pool = rte_calloc(__func__, 1, size, 0);
3149         if (!pool) {
3150                 rte_errno = ENOMEM;
3151                 return NULL;
3152         }
3153         pool->min_dcs = dcs;
3154         pool->raw = cont->init_mem_mng->raws + n_valid %
3155                                                      MLX5_CNT_CONTAINER_RESIZE;
3156         pool->raw_hw = NULL;
3157         rte_spinlock_init(&pool->sl);
3158         /*
3159          * The generation of the new allocated counters in this pool is 0, 2 in
3160          * the pool generation makes all the counters valid for allocation.
3161          */
3162         rte_atomic64_set(&pool->query_gen, 0x2);
3163         TAILQ_INIT(&pool->counters);
3164         TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
3165         cont->pools[n_valid] = pool;
3166         /* Pool initialization must be updated before host thread access. */
3167         rte_cio_wmb();
3168         rte_atomic16_add(&cont->n_valid, 1);
3169         return pool;
3170 }
3171
3172 /**
3173  * Prepare a new counter and/or a new counter pool.
3174  *
3175  * @param[in] dev
3176  *   Pointer to the Ethernet device structure.
3177  * @param[out] cnt_free
3178  *   Where to put the pointer of a new counter.
3179  * @param[in] batch
3180  *   Whether the pool is for counter that was allocated by batch command.
3181  *
3182  * @return
3183  *   The free counter pool pointer and @p cnt_free is set on success,
3184  *   NULL otherwise and rte_errno is set.
3185  */
3186 static struct mlx5_flow_counter_pool *
3187 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
3188                              struct mlx5_flow_counter **cnt_free,
3189                              uint32_t batch)
3190 {
3191         struct mlx5_priv *priv = dev->data->dev_private;
3192         struct mlx5_flow_counter_pool *pool;
3193         struct mlx5_devx_obj *dcs = NULL;
3194         struct mlx5_flow_counter *cnt;
3195         uint32_t i;
3196
3197         if (!batch) {
3198                 /* bulk_bitmap must be 0 for single counter allocation. */
3199                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
3200                 if (!dcs)
3201                         return NULL;
3202                 pool = flow_dv_find_pool_by_id
3203                         (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
3204                 if (!pool) {
3205                         pool = flow_dv_pool_create(dev, dcs, batch);
3206                         if (!pool) {
3207                                 mlx5_devx_cmd_destroy(dcs);
3208                                 return NULL;
3209                         }
3210                 } else if (dcs->id < pool->min_dcs->id) {
3211                         rte_atomic64_set(&pool->a64_dcs,
3212                                          (int64_t)(uintptr_t)dcs);
3213                 }
3214                 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
3215                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
3216                 cnt->dcs = dcs;
3217                 *cnt_free = cnt;
3218                 return pool;
3219         }
3220         /* bulk_bitmap is in 128 counters units. */
3221         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
3222                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
3223         if (!dcs) {
3224                 rte_errno = ENODATA;
3225                 return NULL;
3226         }
3227         pool = flow_dv_pool_create(dev, dcs, batch);
3228         if (!pool) {
3229                 mlx5_devx_cmd_destroy(dcs);
3230                 return NULL;
3231         }
3232         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
3233                 cnt = &pool->counters_raw[i];
3234                 cnt->pool = pool;
3235                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
3236         }
3237         *cnt_free = &pool->counters_raw[0];
3238         return pool;
3239 }
3240
3241 /**
3242  * Search for existed shared counter.
3243  *
3244  * @param[in] cont
3245  *   Pointer to the relevant counter pool container.
3246  * @param[in] id
3247  *   The shared counter ID to search.
3248  *
3249  * @return
3250  *   NULL if not existed, otherwise pointer to the shared counter.
3251  */
3252 static struct mlx5_flow_counter *
3253 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
3254                               uint32_t id)
3255 {
3256         static struct mlx5_flow_counter *cnt;
3257         struct mlx5_flow_counter_pool *pool;
3258         int i;
3259
3260         TAILQ_FOREACH(pool, &cont->pool_list, next) {
3261                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
3262                         cnt = &pool->counters_raw[i];
3263                         if (cnt->ref_cnt && cnt->shared && cnt->id == id)
3264                                 return cnt;
3265                 }
3266         }
3267         return NULL;
3268 }
3269
3270 /**
3271  * Allocate a flow counter.
3272  *
3273  * @param[in] dev
3274  *   Pointer to the Ethernet device structure.
3275  * @param[in] shared
3276  *   Indicate if this counter is shared with other flows.
3277  * @param[in] id
3278  *   Counter identifier.
3279  * @param[in] group
3280  *   Counter flow group.
3281  *
3282  * @return
3283  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
3284  */
3285 static struct mlx5_flow_counter *
3286 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
3287                       uint16_t group)
3288 {
3289         struct mlx5_priv *priv = dev->data->dev_private;
3290         struct mlx5_flow_counter_pool *pool = NULL;
3291         struct mlx5_flow_counter *cnt_free = NULL;
3292         /*
3293          * Currently group 0 flow counter cannot be assigned to a flow if it is
3294          * not the first one in the batch counter allocation, so it is better
3295          * to allocate counters one by one for these flows in a separate
3296          * container.
3297          * A counter can be shared between different groups so need to take
3298          * shared counters from the single container.
3299          */
3300         uint32_t batch = (group && !shared) ? 1 : 0;
3301         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
3302                                                                0);
3303
3304         if (priv->counter_fallback)
3305                 return flow_dv_counter_alloc_fallback(dev, shared, id);
3306         if (!priv->config.devx) {
3307                 rte_errno = ENOTSUP;
3308                 return NULL;
3309         }
3310         if (shared) {
3311                 cnt_free = flow_dv_counter_shared_search(cont, id);
3312                 if (cnt_free) {
3313                         if (cnt_free->ref_cnt + 1 == 0) {
3314                                 rte_errno = E2BIG;
3315                                 return NULL;
3316                         }
3317                         cnt_free->ref_cnt++;
3318                         return cnt_free;
3319                 }
3320         }
3321         /* Pools which has a free counters are in the start. */
3322         TAILQ_FOREACH(pool, &cont->pool_list, next) {
3323                 /*
3324                  * The free counter reset values must be updated between the
3325                  * counter release to the counter allocation, so, at least one
3326                  * query must be done in this time. ensure it by saving the
3327                  * query generation in the release time.
3328                  * The free list is sorted according to the generation - so if
3329                  * the first one is not updated, all the others are not
3330                  * updated too.
3331                  */
3332                 cnt_free = TAILQ_FIRST(&pool->counters);
3333                 if (cnt_free && cnt_free->query_gen + 1 <
3334                     rte_atomic64_read(&pool->query_gen))
3335                         break;
3336                 cnt_free = NULL;
3337         }
3338         if (!cnt_free) {
3339                 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
3340                 if (!pool)
3341                         return NULL;
3342         }
3343         cnt_free->batch = batch;
3344         /* Create a DV counter action only in the first time usage. */
3345         if (!cnt_free->action) {
3346                 uint16_t offset;
3347                 struct mlx5_devx_obj *dcs;
3348
3349                 if (batch) {
3350                         offset = cnt_free - &pool->counters_raw[0];
3351                         dcs = pool->min_dcs;
3352                 } else {
3353                         offset = 0;
3354                         dcs = cnt_free->dcs;
3355                 }
3356                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
3357                                         (dcs->obj, offset);
3358                 if (!cnt_free->action) {
3359                         rte_errno = errno;
3360                         return NULL;
3361                 }
3362         }
3363         /* Update the counter reset values. */
3364         if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
3365                                  &cnt_free->bytes))
3366                 return NULL;
3367         cnt_free->shared = shared;
3368         cnt_free->ref_cnt = 1;
3369         cnt_free->id = id;
3370         if (!priv->sh->cmng.query_thread_on)
3371                 /* Start the asynchronous batch query by the host thread. */
3372                 mlx5_set_query_alarm(priv->sh);
3373         TAILQ_REMOVE(&pool->counters, cnt_free, next);
3374         if (TAILQ_EMPTY(&pool->counters)) {
3375                 /* Move the pool to the end of the container pool list. */
3376                 TAILQ_REMOVE(&cont->pool_list, pool, next);
3377                 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
3378         }
3379         return cnt_free;
3380 }
3381
3382 /**
3383  * Release a flow counter.
3384  *
3385  * @param[in] dev
3386  *   Pointer to the Ethernet device structure.
3387  * @param[in] counter
3388  *   Pointer to the counter handler.
3389  */
3390 static void
3391 flow_dv_counter_release(struct rte_eth_dev *dev,
3392                         struct mlx5_flow_counter *counter)
3393 {
3394         struct mlx5_priv *priv = dev->data->dev_private;
3395
3396         if (!counter)
3397                 return;
3398         if (priv->counter_fallback) {
3399                 flow_dv_counter_release_fallback(dev, counter);
3400                 return;
3401         }
3402         if (--counter->ref_cnt == 0) {
3403                 struct mlx5_flow_counter_pool *pool =
3404                                 flow_dv_counter_pool_get(counter);
3405
3406                 /* Put the counter in the end - the last updated one. */
3407                 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
3408                 counter->query_gen = rte_atomic64_read(&pool->query_gen);
3409         }
3410 }
3411
3412 /**
3413  * Verify the @p attributes will be correctly understood by the NIC and store
3414  * them in the @p flow if everything is correct.
3415  *
3416  * @param[in] dev
3417  *   Pointer to dev struct.
3418  * @param[in] attributes
3419  *   Pointer to flow attributes
3420  * @param[in] external
3421  *   This flow rule is created by request external to PMD.
3422  * @param[out] error
3423  *   Pointer to error structure.
3424  *
3425  * @return
3426  *   0 on success, a negative errno value otherwise and rte_errno is set.
3427  */
3428 static int
3429 flow_dv_validate_attributes(struct rte_eth_dev *dev,
3430                             const struct rte_flow_attr *attributes,
3431                             bool external __rte_unused,
3432                             struct rte_flow_error *error)
3433 {
3434         struct mlx5_priv *priv = dev->data->dev_private;
3435         uint32_t priority_max = priv->config.flow_prio - 1;
3436
3437 #ifndef HAVE_MLX5DV_DR
3438         if (attributes->group)
3439                 return rte_flow_error_set(error, ENOTSUP,
3440                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
3441                                           NULL,
3442                                           "groups are not supported");
3443 #else
3444         uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB :
3445                                                     MLX5_MAX_TABLES;
3446         uint32_t table;
3447         int ret;
3448
3449         ret = mlx5_flow_group_to_table(attributes, external,
3450                                        attributes->group,
3451                                        &table, error);
3452         if (ret)
3453                 return ret;
3454         if (table >= max_group)
3455                 return rte_flow_error_set(error, EINVAL,
3456                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
3457                                           "group index out of range");
3458 #endif
3459         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
3460             attributes->priority >= priority_max)
3461                 return rte_flow_error_set(error, ENOTSUP,
3462                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
3463                                           NULL,
3464                                           "priority out of range");
3465         if (attributes->transfer) {
3466                 if (!priv->config.dv_esw_en)
3467                         return rte_flow_error_set
3468                                 (error, ENOTSUP,
3469                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3470                                  "E-Switch dr is not supported");
3471                 if (!(priv->representor || priv->master))
3472                         return rte_flow_error_set
3473                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3474                                  NULL, "E-Switch configuration can only be"
3475                                  " done by a master or a representor device");
3476                 if (attributes->egress)
3477                         return rte_flow_error_set
3478                                 (error, ENOTSUP,
3479                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
3480                                  "egress is not supported");
3481         }
3482         if (!(attributes->egress ^ attributes->ingress))
3483                 return rte_flow_error_set(error, ENOTSUP,
3484                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
3485                                           "must specify exactly one of "
3486                                           "ingress or egress");
3487         return 0;
3488 }
3489
3490 /**
3491  * Internal validation function. For validating both actions and items.
3492  *
3493  * @param[in] dev
3494  *   Pointer to the rte_eth_dev structure.
3495  * @param[in] attr
3496  *   Pointer to the flow attributes.
3497  * @param[in] items
3498  *   Pointer to the list of items.
3499  * @param[in] actions
3500  *   Pointer to the list of actions.
3501  * @param[in] external
3502  *   This flow rule is created by request external to PMD.
3503  * @param[out] error
3504  *   Pointer to the error structure.
3505  *
3506  * @return
3507  *   0 on success, a negative errno value otherwise and rte_errno is set.
3508  */
3509 static int
3510 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
3511                  const struct rte_flow_item items[],
3512                  const struct rte_flow_action actions[],
3513                  bool external, struct rte_flow_error *error)
3514 {
3515         int ret;
3516         uint64_t action_flags = 0;
3517         uint64_t item_flags = 0;
3518         uint64_t last_item = 0;
3519         uint8_t next_protocol = 0xff;
3520         uint16_t ether_type = 0;
3521         int actions_n = 0;
3522         const struct rte_flow_item *gre_item = NULL;
3523         struct rte_flow_item_tcp nic_tcp_mask = {
3524                 .hdr = {
3525                         .tcp_flags = 0xFF,
3526                         .src_port = RTE_BE16(UINT16_MAX),
3527                         .dst_port = RTE_BE16(UINT16_MAX),
3528                 }
3529         };
3530
3531         if (items == NULL)
3532                 return -1;
3533         ret = flow_dv_validate_attributes(dev, attr, external, error);
3534         if (ret < 0)
3535                 return ret;
3536         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3537                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3538                 int type = items->type;
3539
3540                 switch (type) {
3541                 case RTE_FLOW_ITEM_TYPE_VOID:
3542                         break;
3543                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3544                         ret = flow_dv_validate_item_port_id
3545                                         (dev, items, attr, item_flags, error);
3546                         if (ret < 0)
3547                                 return ret;
3548                         last_item = MLX5_FLOW_ITEM_PORT_ID;
3549                         break;
3550                 case RTE_FLOW_ITEM_TYPE_ETH:
3551                         ret = mlx5_flow_validate_item_eth(items, item_flags,
3552                                                           error);
3553                         if (ret < 0)
3554                                 return ret;
3555                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
3556                                              MLX5_FLOW_LAYER_OUTER_L2;
3557                         if (items->mask != NULL && items->spec != NULL) {
3558                                 ether_type =
3559                                         ((const struct rte_flow_item_eth *)
3560                                          items->spec)->type;
3561                                 ether_type &=
3562                                         ((const struct rte_flow_item_eth *)
3563                                          items->mask)->type;
3564                                 ether_type = rte_be_to_cpu_16(ether_type);
3565                         } else {
3566                                 ether_type = 0;
3567                         }
3568                         break;
3569                 case RTE_FLOW_ITEM_TYPE_VLAN:
3570                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
3571                                                            dev, error);
3572                         if (ret < 0)
3573                                 return ret;
3574                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
3575                                              MLX5_FLOW_LAYER_OUTER_VLAN;
3576                         if (items->mask != NULL && items->spec != NULL) {
3577                                 ether_type =
3578                                         ((const struct rte_flow_item_vlan *)
3579                                          items->spec)->inner_type;
3580                                 ether_type &=
3581                                         ((const struct rte_flow_item_vlan *)
3582                                          items->mask)->inner_type;
3583                                 ether_type = rte_be_to_cpu_16(ether_type);
3584                         } else {
3585                                 ether_type = 0;
3586                         }
3587                         break;
3588                 case RTE_FLOW_ITEM_TYPE_IPV4:
3589                         mlx5_flow_tunnel_ip_check(items, next_protocol,
3590                                                   &item_flags, &tunnel);
3591                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
3592                                                            last_item,
3593                                                            ether_type, NULL,
3594                                                            error);
3595                         if (ret < 0)
3596                                 return ret;
3597                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3598                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3599                         if (items->mask != NULL &&
3600                             ((const struct rte_flow_item_ipv4 *)
3601                              items->mask)->hdr.next_proto_id) {
3602                                 next_protocol =
3603                                         ((const struct rte_flow_item_ipv4 *)
3604                                          (items->spec))->hdr.next_proto_id;
3605                                 next_protocol &=
3606                                         ((const struct rte_flow_item_ipv4 *)
3607                                          (items->mask))->hdr.next_proto_id;
3608                         } else {
3609                                 /* Reset for inner layer. */
3610                                 next_protocol = 0xff;
3611                         }
3612                         break;
3613                 case RTE_FLOW_ITEM_TYPE_IPV6:
3614                         mlx5_flow_tunnel_ip_check(items, next_protocol,
3615                                                   &item_flags, &tunnel);
3616                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
3617                                                            last_item,
3618                                                            ether_type, NULL,
3619                                                            error);
3620                         if (ret < 0)
3621                                 return ret;
3622                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3623                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3624                         if (items->mask != NULL &&
3625                             ((const struct rte_flow_item_ipv6 *)
3626                              items->mask)->hdr.proto) {
3627                                 next_protocol =
3628                                         ((const struct rte_flow_item_ipv6 *)
3629                                          items->spec)->hdr.proto;
3630                                 next_protocol &=
3631                                         ((const struct rte_flow_item_ipv6 *)
3632                                          items->mask)->hdr.proto;
3633                         } else {
3634                                 /* Reset for inner layer. */
3635                                 next_protocol = 0xff;
3636                         }
3637                         break;
3638                 case RTE_FLOW_ITEM_TYPE_TCP:
3639                         ret = mlx5_flow_validate_item_tcp
3640                                                 (items, item_flags,
3641                                                  next_protocol,
3642                                                  &nic_tcp_mask,
3643                                                  error);
3644                         if (ret < 0)
3645                                 return ret;
3646                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3647                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
3648                         break;
3649                 case RTE_FLOW_ITEM_TYPE_UDP:
3650                         ret = mlx5_flow_validate_item_udp(items, item_flags,
3651                                                           next_protocol,
3652                                                           error);
3653                         if (ret < 0)
3654                                 return ret;
3655                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3656                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
3657                         break;
3658                 case RTE_FLOW_ITEM_TYPE_GRE:
3659                         ret = mlx5_flow_validate_item_gre(items, item_flags,
3660                                                           next_protocol, error);
3661                         if (ret < 0)
3662                                 return ret;
3663                         gre_item = items;
3664                         last_item = MLX5_FLOW_LAYER_GRE;
3665                         break;
3666                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3667                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
3668                                                             next_protocol,
3669                                                             error);
3670                         if (ret < 0)
3671                                 return ret;
3672                         last_item = MLX5_FLOW_LAYER_NVGRE;
3673                         break;
3674                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
3675                         ret = mlx5_flow_validate_item_gre_key
3676                                 (items, item_flags, gre_item, error);
3677                         if (ret < 0)
3678                                 return ret;
3679                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
3680                         break;
3681                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3682                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
3683                                                             error);
3684                         if (ret < 0)
3685                                 return ret;
3686                         last_item = MLX5_FLOW_LAYER_VXLAN;
3687                         break;
3688                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3689                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
3690                                                                 item_flags, dev,
3691                                                                 error);
3692                         if (ret < 0)
3693                                 return ret;
3694                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3695                         break;
3696                 case RTE_FLOW_ITEM_TYPE_GENEVE:
3697                         ret = mlx5_flow_validate_item_geneve(items,
3698                                                              item_flags, dev,
3699                                                              error);
3700                         if (ret < 0)
3701                                 return ret;
3702                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3703                         break;
3704                 case RTE_FLOW_ITEM_TYPE_MPLS:
3705                         ret = mlx5_flow_validate_item_mpls(dev, items,
3706                                                            item_flags,
3707                                                            last_item, error);
3708                         if (ret < 0)
3709                                 return ret;
3710                         last_item = MLX5_FLOW_LAYER_MPLS;
3711                         break;
3712                 case RTE_FLOW_ITEM_TYPE_META:
3713                         ret = flow_dv_validate_item_meta(dev, items, attr,
3714                                                          error);
3715                         if (ret < 0)
3716                                 return ret;
3717                         last_item = MLX5_FLOW_ITEM_METADATA;
3718                         break;
3719                 case RTE_FLOW_ITEM_TYPE_ICMP:
3720                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
3721                                                            next_protocol,
3722                                                            error);
3723                         if (ret < 0)
3724                                 return ret;
3725                         last_item = MLX5_FLOW_LAYER_ICMP;
3726                         break;
3727                 case RTE_FLOW_ITEM_TYPE_ICMP6:
3728                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3729                                                             next_protocol,
3730                                                             error);
3731                         if (ret < 0)
3732                                 return ret;
3733                         last_item = MLX5_FLOW_LAYER_ICMP6;
3734                         break;
3735                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
3736                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
3737                         break;
3738                 default:
3739                         return rte_flow_error_set(error, ENOTSUP,
3740                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3741                                                   NULL, "item not supported");
3742                 }
3743                 item_flags |= last_item;
3744         }
3745         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3746                 int type = actions->type;
3747                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3748                         return rte_flow_error_set(error, ENOTSUP,
3749                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3750                                                   actions, "too many actions");
3751                 switch (type) {
3752                 case RTE_FLOW_ACTION_TYPE_VOID:
3753                         break;
3754                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3755                         ret = flow_dv_validate_action_port_id(dev,
3756                                                               action_flags,
3757                                                               actions,
3758                                                               attr,
3759                                                               error);
3760                         if (ret)
3761                                 return ret;
3762                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3763                         ++actions_n;
3764                         break;
3765                 case RTE_FLOW_ACTION_TYPE_FLAG:
3766                         ret = mlx5_flow_validate_action_flag(action_flags,
3767                                                              attr, error);
3768                         if (ret < 0)
3769                                 return ret;
3770                         action_flags |= MLX5_FLOW_ACTION_FLAG;
3771                         ++actions_n;
3772                         break;
3773                 case RTE_FLOW_ACTION_TYPE_MARK:
3774                         ret = mlx5_flow_validate_action_mark(actions,
3775                                                              action_flags,
3776                                                              attr, error);
3777                         if (ret < 0)
3778                                 return ret;
3779                         action_flags |= MLX5_FLOW_ACTION_MARK;
3780                         ++actions_n;
3781                         break;
3782                 case RTE_FLOW_ACTION_TYPE_DROP:
3783                         ret = mlx5_flow_validate_action_drop(action_flags,
3784                                                              attr, error);
3785                         if (ret < 0)
3786                                 return ret;
3787                         action_flags |= MLX5_FLOW_ACTION_DROP;
3788                         ++actions_n;
3789                         break;
3790                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3791                         ret = mlx5_flow_validate_action_queue(actions,
3792                                                               action_flags, dev,
3793                                                               attr, error);
3794                         if (ret < 0)
3795                                 return ret;
3796                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
3797                         ++actions_n;
3798                         break;
3799                 case RTE_FLOW_ACTION_TYPE_RSS:
3800                         ret = mlx5_flow_validate_action_rss(actions,
3801                                                             action_flags, dev,
3802                                                             attr, item_flags,
3803                                                             error);
3804                         if (ret < 0)
3805                                 return ret;
3806                         action_flags |= MLX5_FLOW_ACTION_RSS;
3807                         ++actions_n;
3808                         break;
3809                 case RTE_FLOW_ACTION_TYPE_COUNT:
3810                         ret = flow_dv_validate_action_count(dev, error);
3811                         if (ret < 0)
3812                                 return ret;
3813                         action_flags |= MLX5_FLOW_ACTION_COUNT;
3814                         ++actions_n;
3815                         break;
3816                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3817                         if (flow_dv_validate_action_pop_vlan(dev,
3818                                                              action_flags,
3819                                                              actions,
3820                                                              item_flags, attr,
3821                                                              error))
3822                                 return -rte_errno;
3823                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
3824                         ++actions_n;
3825                         break;
3826                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3827                         ret = flow_dv_validate_action_push_vlan(action_flags,
3828                                                                 item_flags,
3829                                                                 actions, attr,
3830                                                                 error);
3831                         if (ret < 0)
3832                                 return ret;
3833                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
3834                         ++actions_n;
3835                         break;
3836                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3837                         ret = flow_dv_validate_action_set_vlan_pcp
3838                                                 (action_flags, actions, error);
3839                         if (ret < 0)
3840                                 return ret;
3841                         /* Count PCP with push_vlan command. */
3842                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
3843                         break;
3844                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3845                         ret = flow_dv_validate_action_set_vlan_vid
3846                                                 (item_flags, action_flags,
3847                                                  actions, error);
3848                         if (ret < 0)
3849                                 return ret;
3850                         /* Count VID with push_vlan command. */
3851                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
3852                         break;
3853                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3854                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3855                         ret = flow_dv_validate_action_l2_encap(action_flags,
3856                                                                actions, attr,
3857                                                                error);
3858                         if (ret < 0)
3859                                 return ret;
3860                         action_flags |= actions->type ==
3861                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3862                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
3863                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
3864                         ++actions_n;
3865                         break;
3866                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3867                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3868                         ret = flow_dv_validate_action_l2_decap(action_flags,
3869                                                                attr, error);
3870                         if (ret < 0)
3871                                 return ret;
3872                         action_flags |= actions->type ==
3873                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3874                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
3875                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
3876                         ++actions_n;
3877                         break;
3878                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3879                         ret = flow_dv_validate_action_raw_encap(action_flags,
3880                                                                 actions, attr,
3881                                                                 error);
3882                         if (ret < 0)
3883                                 return ret;
3884                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3885                         ++actions_n;
3886                         break;
3887                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3888                         ret = flow_dv_validate_action_raw_decap(action_flags,
3889                                                                 actions, attr,
3890                                                                 error);
3891                         if (ret < 0)
3892                                 return ret;
3893                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3894                         ++actions_n;
3895                         break;
3896                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3897                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3898                         ret = flow_dv_validate_action_modify_mac(action_flags,
3899                                                                  actions,
3900                                                                  item_flags,
3901                                                                  error);
3902                         if (ret < 0)
3903                                 return ret;
3904                         /* Count all modify-header actions as one action. */
3905                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3906                                 ++actions_n;
3907                         action_flags |= actions->type ==
3908                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3909                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
3910                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
3911                         break;
3912
3913                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3914                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3915                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
3916                                                                   actions,
3917                                                                   item_flags,
3918                                                                   error);
3919                         if (ret < 0)
3920                                 return ret;
3921                         /* Count all modify-header actions as one action. */
3922                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3923                                 ++actions_n;
3924                         action_flags |= actions->type ==
3925                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3926                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3927                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
3928                         break;
3929                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3930                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3931                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
3932                                                                   actions,
3933                                                                   item_flags,
3934                                                                   error);
3935                         if (ret < 0)
3936                                 return ret;
3937                         /* Count all modify-header actions as one action. */
3938                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3939                                 ++actions_n;
3940                         action_flags |= actions->type ==
3941                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3942                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3943                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
3944                         break;
3945                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3946                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3947                         ret = flow_dv_validate_action_modify_tp(action_flags,
3948                                                                 actions,
3949                                                                 item_flags,
3950                                                                 error);
3951                         if (ret < 0)
3952                                 return ret;
3953                         /* Count all modify-header actions as one action. */
3954                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3955                                 ++actions_n;
3956                         action_flags |= actions->type ==
3957                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3958                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
3959                                                 MLX5_FLOW_ACTION_SET_TP_DST;
3960                         break;
3961                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3962                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3963                         ret = flow_dv_validate_action_modify_ttl(action_flags,
3964                                                                  actions,
3965                                                                  item_flags,
3966                                                                  error);
3967                         if (ret < 0)
3968                                 return ret;
3969                         /* Count all modify-header actions as one action. */
3970                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3971                                 ++actions_n;
3972                         action_flags |= actions->type ==
3973                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
3974                                                 MLX5_FLOW_ACTION_SET_TTL :
3975                                                 MLX5_FLOW_ACTION_DEC_TTL;
3976                         break;
3977                 case RTE_FLOW_ACTION_TYPE_JUMP:
3978                         ret = flow_dv_validate_action_jump(actions,
3979                                                            action_flags,
3980                                                            attr, external,
3981                                                            error);
3982                         if (ret)
3983                                 return ret;
3984                         ++actions_n;
3985                         action_flags |= MLX5_FLOW_ACTION_JUMP;
3986                         break;
3987                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3988                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3989                         ret = flow_dv_validate_action_modify_tcp_seq
3990                                                                 (action_flags,
3991                                                                  actions,
3992                                                                  item_flags,
3993                                                                  error);
3994                         if (ret < 0)
3995                                 return ret;
3996                         /* Count all modify-header actions as one action. */
3997                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3998                                 ++actions_n;
3999                         action_flags |= actions->type ==
4000                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
4001                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
4002                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
4003                         break;
4004                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4005                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4006                         ret = flow_dv_validate_action_modify_tcp_ack
4007                                                                 (action_flags,
4008                                                                  actions,
4009                                                                  item_flags,
4010                                                                  error);
4011                         if (ret < 0)
4012                                 return ret;
4013                         /* Count all modify-header actions as one action. */
4014                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
4015                                 ++actions_n;
4016                         action_flags |= actions->type ==
4017                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
4018                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
4019                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
4020                         break;
4021                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
4022                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
4023                         break;
4024                 default:
4025                         return rte_flow_error_set(error, ENOTSUP,
4026                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4027                                                   actions,
4028                                                   "action not supported");
4029                 }
4030         }
4031         if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) &&
4032             (action_flags & MLX5_FLOW_VLAN_ACTIONS))
4033                 return rte_flow_error_set(error, ENOTSUP,
4034                                           RTE_FLOW_ERROR_TYPE_ACTION,
4035                                           actions,
4036                                           "can't have vxlan and vlan"
4037                                           " actions in the same rule");
4038         /* Eswitch has few restrictions on using items and actions */
4039         if (attr->transfer) {
4040                 if (action_flags & MLX5_FLOW_ACTION_FLAG)
4041                         return rte_flow_error_set(error, ENOTSUP,
4042                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4043                                                   NULL,
4044                                                   "unsupported action FLAG");
4045                 if (action_flags & MLX5_FLOW_ACTION_MARK)
4046                         return rte_flow_error_set(error, ENOTSUP,
4047                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4048                                                   NULL,
4049                                                   "unsupported action MARK");
4050                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
4051                         return rte_flow_error_set(error, ENOTSUP,
4052                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4053                                                   NULL,
4054                                                   "unsupported action QUEUE");
4055                 if (action_flags & MLX5_FLOW_ACTION_RSS)
4056                         return rte_flow_error_set(error, ENOTSUP,
4057                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4058                                                   NULL,
4059                                                   "unsupported action RSS");
4060                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4061                         return rte_flow_error_set(error, EINVAL,
4062                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4063                                                   actions,
4064                                                   "no fate action is found");
4065         } else {
4066                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
4067                         return rte_flow_error_set(error, EINVAL,
4068                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4069                                                   actions,
4070                                                   "no fate action is found");
4071         }
4072         return 0;
4073 }
4074
4075 /**
4076  * Internal preparation function. Allocates the DV flow size,
4077  * this size is constant.
4078  *
4079  * @param[in] attr
4080  *   Pointer to the flow attributes.
4081  * @param[in] items
4082  *   Pointer to the list of items.
4083  * @param[in] actions
4084  *   Pointer to the list of actions.
4085  * @param[out] error
4086  *   Pointer to the error structure.
4087  *
4088  * @return
4089  *   Pointer to mlx5_flow object on success,
4090  *   otherwise NULL and rte_errno is set.
4091  */
4092 static struct mlx5_flow *
4093 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
4094                 const struct rte_flow_item items[] __rte_unused,
4095                 const struct rte_flow_action actions[] __rte_unused,
4096                 struct rte_flow_error *error)
4097 {
4098         size_t size = sizeof(struct mlx5_flow);
4099         struct mlx5_flow *dev_flow;
4100
4101         dev_flow = rte_calloc(__func__, 1, size, 0);
4102         if (!dev_flow) {
4103                 rte_flow_error_set(error, ENOMEM,
4104                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4105                                    "not enough memory to create flow");
4106                 return NULL;
4107         }
4108         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
4109         dev_flow->ingress = attr->ingress;
4110         dev_flow->transfer = attr->transfer;
4111         return dev_flow;
4112 }
4113
4114 #ifndef NDEBUG
4115 /**
4116  * Sanity check for match mask and value. Similar to check_valid_spec() in
4117  * kernel driver. If unmasked bit is present in value, it returns failure.
4118  *
4119  * @param match_mask
4120  *   pointer to match mask buffer.
4121  * @param match_value
4122  *   pointer to match value buffer.
4123  *
4124  * @return
4125  *   0 if valid, -EINVAL otherwise.
4126  */
4127 static int
4128 flow_dv_check_valid_spec(void *match_mask, void *match_value)
4129 {
4130         uint8_t *m = match_mask;
4131         uint8_t *v = match_value;
4132         unsigned int i;
4133
4134         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
4135                 if (v[i] & ~m[i]) {
4136                         DRV_LOG(ERR,
4137                                 "match_value differs from match_criteria"
4138                                 " %p[%u] != %p[%u]",
4139                                 match_value, i, match_mask, i);
4140                         return -EINVAL;
4141                 }
4142         }
4143         return 0;
4144 }
4145 #endif
4146
4147 /**
4148  * Add Ethernet item to matcher and to the value.
4149  *
4150  * @param[in, out] matcher
4151  *   Flow matcher.
4152  * @param[in, out] key
4153  *   Flow matcher value.
4154  * @param[in] item
4155  *   Flow pattern to translate.
4156  * @param[in] inner
4157  *   Item is inner pattern.
4158  */
4159 static void
4160 flow_dv_translate_item_eth(void *matcher, void *key,
4161                            const struct rte_flow_item *item, int inner)
4162 {
4163         const struct rte_flow_item_eth *eth_m = item->mask;
4164         const struct rte_flow_item_eth *eth_v = item->spec;
4165         const struct rte_flow_item_eth nic_mask = {
4166                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
4167                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
4168                 .type = RTE_BE16(0xffff),
4169         };
4170         void *headers_m;
4171         void *headers_v;
4172         char *l24_v;
4173         unsigned int i;
4174
4175         if (!eth_v)
4176                 return;
4177         if (!eth_m)
4178                 eth_m = &nic_mask;
4179         if (inner) {
4180                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4181                                          inner_headers);
4182                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4183         } else {
4184                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4185                                          outer_headers);
4186                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4187         }
4188         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
4189                &eth_m->dst, sizeof(eth_m->dst));
4190         /* The value must be in the range of the mask. */
4191         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
4192         for (i = 0; i < sizeof(eth_m->dst); ++i)
4193                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
4194         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
4195                &eth_m->src, sizeof(eth_m->src));
4196         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
4197         /* The value must be in the range of the mask. */
4198         for (i = 0; i < sizeof(eth_m->dst); ++i)
4199                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
4200         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
4201                  rte_be_to_cpu_16(eth_m->type));
4202         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
4203         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
4204 }
4205
4206 /**
4207  * Add VLAN item to matcher and to the value.
4208  *
4209  * @param[in, out] dev_flow
4210  *   Flow descriptor.
4211  * @param[in, out] matcher
4212  *   Flow matcher.
4213  * @param[in, out] key
4214  *   Flow matcher value.
4215  * @param[in] item
4216  *   Flow pattern to translate.
4217  * @param[in] inner
4218  *   Item is inner pattern.
4219  */
4220 static void
4221 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
4222                             void *matcher, void *key,
4223                             const struct rte_flow_item *item,
4224                             int inner)
4225 {
4226         const struct rte_flow_item_vlan *vlan_m = item->mask;
4227         const struct rte_flow_item_vlan *vlan_v = item->spec;
4228         void *headers_m;
4229         void *headers_v;
4230         uint16_t tci_m;
4231         uint16_t tci_v;
4232
4233         if (!vlan_v)
4234                 return;
4235         if (!vlan_m)
4236                 vlan_m = &rte_flow_item_vlan_mask;
4237         if (inner) {
4238                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4239                                          inner_headers);
4240                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4241         } else {
4242                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4243                                          outer_headers);
4244                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4245                 /*
4246                  * This is workaround, masks are not supported,
4247                  * and pre-validated.
4248                  */
4249                 dev_flow->dv.vf_vlan.tag =
4250                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
4251         }
4252         tci_m = rte_be_to_cpu_16(vlan_m->tci);
4253         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
4254         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
4255         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
4256         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
4257         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
4258         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
4259         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
4260         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
4261         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
4262         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
4263                  rte_be_to_cpu_16(vlan_m->inner_type));
4264         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
4265                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
4266 }
4267
4268 /**
4269  * Add IPV4 item to matcher and to the value.
4270  *
4271  * @param[in, out] matcher
4272  *   Flow matcher.
4273  * @param[in, out] key
4274  *   Flow matcher value.
4275  * @param[in] item
4276  *   Flow pattern to translate.
4277  * @param[in] inner
4278  *   Item is inner pattern.
4279  * @param[in] group
4280  *   The group to insert the rule.
4281  */
4282 static void
4283 flow_dv_translate_item_ipv4(void *matcher, void *key,
4284                             const struct rte_flow_item *item,
4285                             int inner, uint32_t group)
4286 {
4287         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
4288         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
4289         const struct rte_flow_item_ipv4 nic_mask = {
4290                 .hdr = {
4291                         .src_addr = RTE_BE32(0xffffffff),
4292                         .dst_addr = RTE_BE32(0xffffffff),
4293                         .type_of_service = 0xff,
4294                         .next_proto_id = 0xff,
4295                 },
4296         };
4297         void *headers_m;
4298         void *headers_v;
4299         char *l24_m;
4300         char *l24_v;
4301         uint8_t tos;
4302
4303         if (inner) {
4304                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4305                                          inner_headers);
4306                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4307         } else {
4308                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4309                                          outer_headers);
4310                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4311         }
4312         if (group == 0)
4313                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
4314         else
4315                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
4316         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
4317         if (!ipv4_v)
4318                 return;
4319         if (!ipv4_m)
4320                 ipv4_m = &nic_mask;
4321         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4322                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
4323         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4324                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
4325         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
4326         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
4327         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4328                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
4329         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4330                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
4331         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
4332         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
4333         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
4334         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
4335                  ipv4_m->hdr.type_of_service);
4336         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
4337         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
4338                  ipv4_m->hdr.type_of_service >> 2);
4339         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
4340         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
4341                  ipv4_m->hdr.next_proto_id);
4342         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4343                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
4344 }
4345
4346 /**
4347  * Add IPV6 item to matcher and to the value.
4348  *
4349  * @param[in, out] matcher
4350  *   Flow matcher.
4351  * @param[in, out] key
4352  *   Flow matcher value.
4353  * @param[in] item
4354  *   Flow pattern to translate.
4355  * @param[in] inner
4356  *   Item is inner pattern.
4357  * @param[in] group
4358  *   The group to insert the rule.
4359  */
4360 static void
4361 flow_dv_translate_item_ipv6(void *matcher, void *key,
4362                             const struct rte_flow_item *item,
4363                             int inner, uint32_t group)
4364 {
4365         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
4366         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
4367         const struct rte_flow_item_ipv6 nic_mask = {
4368                 .hdr = {
4369                         .src_addr =
4370                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
4371                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
4372                         .dst_addr =
4373                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
4374                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
4375                         .vtc_flow = RTE_BE32(0xffffffff),
4376                         .proto = 0xff,
4377                         .hop_limits = 0xff,
4378                 },
4379         };
4380         void *headers_m;
4381         void *headers_v;
4382         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4383         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4384         char *l24_m;
4385         char *l24_v;
4386         uint32_t vtc_m;
4387         uint32_t vtc_v;
4388         int i;
4389         int size;
4390
4391         if (inner) {
4392                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4393                                          inner_headers);
4394                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4395         } else {
4396                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4397                                          outer_headers);
4398                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4399         }
4400         if (group == 0)
4401                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
4402         else
4403                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
4404         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
4405         if (!ipv6_v)
4406                 return;
4407         if (!ipv6_m)
4408                 ipv6_m = &nic_mask;
4409         size = sizeof(ipv6_m->hdr.dst_addr);
4410         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4411                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
4412         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4413                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
4414         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
4415         for (i = 0; i < size; ++i)
4416                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
4417         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4418                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
4419         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4420                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
4421         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
4422         for (i = 0; i < size; ++i)
4423                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
4424         /* TOS. */
4425         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
4426         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
4427         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
4428         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
4429         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
4430         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
4431         /* Label. */
4432         if (inner) {
4433                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
4434                          vtc_m);
4435                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
4436                          vtc_v);
4437         } else {
4438                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
4439                          vtc_m);
4440                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
4441                          vtc_v);
4442         }
4443         /* Protocol. */
4444         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
4445                  ipv6_m->hdr.proto);
4446         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4447                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
4448 }
4449
4450 /**
4451  * Add TCP item to matcher and to the value.
4452  *
4453  * @param[in, out] matcher
4454  *   Flow matcher.
4455  * @param[in, out] key
4456  *   Flow matcher value.
4457  * @param[in] item
4458  *   Flow pattern to translate.
4459  * @param[in] inner
4460  *   Item is inner pattern.
4461  */
4462 static void
4463 flow_dv_translate_item_tcp(void *matcher, void *key,
4464                            const struct rte_flow_item *item,
4465                            int inner)
4466 {
4467         const struct rte_flow_item_tcp *tcp_m = item->mask;
4468         const struct rte_flow_item_tcp *tcp_v = item->spec;
4469         void *headers_m;
4470         void *headers_v;
4471
4472         if (inner) {
4473                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4474                                          inner_headers);
4475                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4476         } else {
4477                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4478                                          outer_headers);
4479                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4480         }
4481         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4482         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
4483         if (!tcp_v)
4484                 return;
4485         if (!tcp_m)
4486                 tcp_m = &rte_flow_item_tcp_mask;
4487         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
4488                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
4489         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
4490                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
4491         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
4492                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
4493         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
4494                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
4495         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
4496                  tcp_m->hdr.tcp_flags);
4497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
4498                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
4499 }
4500
4501 /**
4502  * Add UDP item to matcher and to the value.
4503  *
4504  * @param[in, out] matcher
4505  *   Flow matcher.
4506  * @param[in, out] key
4507  *   Flow matcher value.
4508  * @param[in] item
4509  *   Flow pattern to translate.
4510  * @param[in] inner
4511  *   Item is inner pattern.
4512  */
4513 static void
4514 flow_dv_translate_item_udp(void *matcher, void *key,
4515                            const struct rte_flow_item *item,
4516                            int inner)
4517 {
4518         const struct rte_flow_item_udp *udp_m = item->mask;
4519         const struct rte_flow_item_udp *udp_v = item->spec;
4520         void *headers_m;
4521         void *headers_v;
4522
4523         if (inner) {
4524                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4525                                          inner_headers);
4526                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4527         } else {
4528                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4529                                          outer_headers);
4530                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4531         }
4532         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4533         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
4534         if (!udp_v)
4535                 return;
4536         if (!udp_m)
4537                 udp_m = &rte_flow_item_udp_mask;
4538         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
4539                  rte_be_to_cpu_16(udp_m->hdr.src_port));
4540         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
4541                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
4542         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
4543                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
4544         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4545                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
4546 }
4547
4548 /**
4549  * Add GRE optional Key item to matcher and to the value.
4550  *
4551  * @param[in, out] matcher
4552  *   Flow matcher.
4553  * @param[in, out] key
4554  *   Flow matcher value.
4555  * @param[in] item
4556  *   Flow pattern to translate.
4557  * @param[in] inner
4558  *   Item is inner pattern.
4559  */
4560 static void
4561 flow_dv_translate_item_gre_key(void *matcher, void *key,
4562                                    const struct rte_flow_item *item)
4563 {
4564         const rte_be32_t *key_m = item->mask;
4565         const rte_be32_t *key_v = item->spec;
4566         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4567         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4568         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
4569
4570         if (!key_v)
4571                 return;
4572         if (!key_m)
4573                 key_m = &gre_key_default_mask;
4574         /* GRE K bit must be on and should already be validated */
4575         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
4576         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
4577         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
4578                  rte_be_to_cpu_32(*key_m) >> 8);
4579         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
4580                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
4581         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
4582                  rte_be_to_cpu_32(*key_m) & 0xFF);
4583         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
4584                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
4585 }
4586
4587 /**
4588  * Add GRE item to matcher and to the value.
4589  *
4590  * @param[in, out] matcher
4591  *   Flow matcher.
4592  * @param[in, out] key
4593  *   Flow matcher value.
4594  * @param[in] item
4595  *   Flow pattern to translate.
4596  * @param[in] inner
4597  *   Item is inner pattern.
4598  */
4599 static void
4600 flow_dv_translate_item_gre(void *matcher, void *key,
4601                            const struct rte_flow_item *item,
4602                            int inner)
4603 {
4604         const struct rte_flow_item_gre *gre_m = item->mask;
4605         const struct rte_flow_item_gre *gre_v = item->spec;
4606         void *headers_m;
4607         void *headers_v;
4608         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4609         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4610         struct {
4611                 union {
4612                         __extension__
4613                         struct {
4614                                 uint16_t version:3;
4615                                 uint16_t rsvd0:9;
4616                                 uint16_t s_present:1;
4617                                 uint16_t k_present:1;
4618                                 uint16_t rsvd_bit1:1;
4619                                 uint16_t c_present:1;
4620                         };
4621                         uint16_t value;
4622                 };
4623         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
4624
4625         if (inner) {
4626                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4627                                          inner_headers);
4628                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4629         } else {
4630                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4631                                          outer_headers);
4632                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4633         }
4634         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4635         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
4636         if (!gre_v)
4637                 return;
4638         if (!gre_m)
4639                 gre_m = &rte_flow_item_gre_mask;
4640         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
4641                  rte_be_to_cpu_16(gre_m->protocol));
4642         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4643                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
4644         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
4645         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
4646         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
4647                  gre_crks_rsvd0_ver_m.c_present);
4648         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
4649                  gre_crks_rsvd0_ver_v.c_present &
4650                  gre_crks_rsvd0_ver_m.c_present);
4651         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
4652                  gre_crks_rsvd0_ver_m.k_present);
4653         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
4654                  gre_crks_rsvd0_ver_v.k_present &
4655                  gre_crks_rsvd0_ver_m.k_present);
4656         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
4657                  gre_crks_rsvd0_ver_m.s_present);
4658         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
4659                  gre_crks_rsvd0_ver_v.s_present &
4660                  gre_crks_rsvd0_ver_m.s_present);
4661 }
4662
4663 /**
4664  * Add NVGRE item to matcher and to the value.
4665  *
4666  * @param[in, out] matcher
4667  *   Flow matcher.
4668  * @param[in, out] key
4669  *   Flow matcher value.
4670  * @param[in] item
4671  *   Flow pattern to translate.
4672  * @param[in] inner
4673  *   Item is inner pattern.
4674  */
4675 static void
4676 flow_dv_translate_item_nvgre(void *matcher, void *key,
4677                              const struct rte_flow_item *item,
4678                              int inner)
4679 {
4680         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
4681         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
4682         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4683         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4684         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
4685         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
4686         char *gre_key_m;
4687         char *gre_key_v;
4688         int size;
4689         int i;
4690
4691         /* For NVGRE, GRE header fields must be set with defined values. */
4692         const struct rte_flow_item_gre gre_spec = {
4693                 .c_rsvd0_ver = RTE_BE16(0x2000),
4694                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
4695         };
4696         const struct rte_flow_item_gre gre_mask = {
4697                 .c_rsvd0_ver = RTE_BE16(0xB000),
4698                 .protocol = RTE_BE16(UINT16_MAX),
4699         };
4700         const struct rte_flow_item gre_item = {
4701                 .spec = &gre_spec,
4702                 .mask = &gre_mask,
4703                 .last = NULL,
4704         };
4705         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
4706         if (!nvgre_v)
4707                 return;
4708         if (!nvgre_m)
4709                 nvgre_m = &rte_flow_item_nvgre_mask;
4710         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
4711         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
4712         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
4713         memcpy(gre_key_m, tni_flow_id_m, size);
4714         for (i = 0; i < size; ++i)
4715                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
4716 }
4717
4718 /**
4719  * Add VXLAN item to matcher and to the value.
4720  *
4721  * @param[in, out] matcher
4722  *   Flow matcher.
4723  * @param[in, out] key
4724  *   Flow matcher value.
4725  * @param[in] item
4726  *   Flow pattern to translate.
4727  * @param[in] inner
4728  *   Item is inner pattern.
4729  */
4730 static void
4731 flow_dv_translate_item_vxlan(void *matcher, void *key,
4732                              const struct rte_flow_item *item,
4733                              int inner)
4734 {
4735         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
4736         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
4737         void *headers_m;
4738         void *headers_v;
4739         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4740         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4741         char *vni_m;
4742         char *vni_v;
4743         uint16_t dport;
4744         int size;
4745         int i;
4746
4747         if (inner) {
4748                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4749                                          inner_headers);
4750                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4751         } else {
4752                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4753                                          outer_headers);
4754                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4755         }
4756         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
4757                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
4758         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
4759                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
4760                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
4761         }
4762         if (!vxlan_v)
4763                 return;
4764         if (!vxlan_m)
4765                 vxlan_m = &rte_flow_item_vxlan_mask;
4766         size = sizeof(vxlan_m->vni);
4767         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4768         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4769         memcpy(vni_m, vxlan_m->vni, size);
4770         for (i = 0; i < size; ++i)
4771                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4772 }
4773
4774 /**
4775  * Add Geneve item to matcher and to the value.
4776  *
4777  * @param[in, out] matcher
4778  *   Flow matcher.
4779  * @param[in, out] key
4780  *   Flow matcher value.
4781  * @param[in] item
4782  *   Flow pattern to translate.
4783  * @param[in] inner
4784  *   Item is inner pattern.
4785  */
4786
4787 static void
4788 flow_dv_translate_item_geneve(void *matcher, void *key,
4789                               const struct rte_flow_item *item, int inner)
4790 {
4791         const struct rte_flow_item_geneve *geneve_m = item->mask;
4792         const struct rte_flow_item_geneve *geneve_v = item->spec;
4793         void *headers_m;
4794         void *headers_v;
4795         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4796         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4797         uint16_t dport;
4798         uint16_t gbhdr_m;
4799         uint16_t gbhdr_v;
4800         char *vni_m;
4801         char *vni_v;
4802         size_t size, i;
4803
4804         if (inner) {
4805                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4806                                          inner_headers);
4807                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4808         } else {
4809                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4810                                          outer_headers);
4811                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4812         }
4813         dport = MLX5_UDP_PORT_GENEVE;
4814         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
4815                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
4816                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
4817         }
4818         if (!geneve_v)
4819                 return;
4820         if (!geneve_m)
4821                 geneve_m = &rte_flow_item_geneve_mask;
4822         size = sizeof(geneve_m->vni);
4823         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
4824         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
4825         memcpy(vni_m, geneve_m->vni, size);
4826         for (i = 0; i < size; ++i)
4827                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
4828         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
4829                  rte_be_to_cpu_16(geneve_m->protocol));
4830         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
4831                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
4832         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
4833         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
4834         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
4835                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
4836         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
4837                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
4838         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
4839                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
4840         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
4841                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
4842                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
4843 }
4844
4845 /**
4846  * Add MPLS item to matcher and to the value.
4847  *
4848  * @param[in, out] matcher
4849  *   Flow matcher.
4850  * @param[in, out] key
4851  *   Flow matcher value.
4852  * @param[in] item
4853  *   Flow pattern to translate.
4854  * @param[in] prev_layer
4855  *   The protocol layer indicated in previous item.
4856  * @param[in] inner
4857  *   Item is inner pattern.
4858  */
4859 static void
4860 flow_dv_translate_item_mpls(void *matcher, void *key,
4861                             const struct rte_flow_item *item,
4862                             uint64_t prev_layer,
4863                             int inner)
4864 {
4865         const uint32_t *in_mpls_m = item->mask;
4866         const uint32_t *in_mpls_v = item->spec;
4867         uint32_t *out_mpls_m = 0;
4868         uint32_t *out_mpls_v = 0;
4869         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4870         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4871         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4872                                      misc_parameters_2);
4873         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4874         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4875         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4876
4877         switch (prev_layer) {
4878         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4879                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4880                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4881                          MLX5_UDP_PORT_MPLS);
4882                 break;
4883         case MLX5_FLOW_LAYER_GRE:
4884                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4885                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4886                          RTE_ETHER_TYPE_MPLS);
4887                 break;
4888         default:
4889                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4890                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4891                          IPPROTO_MPLS);
4892                 break;
4893         }
4894         if (!in_mpls_v)
4895                 return;
4896         if (!in_mpls_m)
4897                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4898         switch (prev_layer) {
4899         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4900                 out_mpls_m =
4901                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4902                                                  outer_first_mpls_over_udp);
4903                 out_mpls_v =
4904                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4905                                                  outer_first_mpls_over_udp);
4906                 break;
4907         case MLX5_FLOW_LAYER_GRE:
4908                 out_mpls_m =
4909                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4910                                                  outer_first_mpls_over_gre);
4911                 out_mpls_v =
4912                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4913                                                  outer_first_mpls_over_gre);
4914                 break;
4915         default:
4916                 /* Inner MPLS not over GRE is not supported. */
4917                 if (!inner) {
4918                         out_mpls_m =
4919                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4920                                                          misc2_m,
4921                                                          outer_first_mpls);
4922                         out_mpls_v =
4923                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4924                                                          misc2_v,
4925                                                          outer_first_mpls);
4926                 }
4927                 break;
4928         }
4929         if (out_mpls_m && out_mpls_v) {
4930                 *out_mpls_m = *in_mpls_m;
4931                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4932         }
4933 }
4934
4935 /**
4936  * Add metadata register item to matcher
4937  *
4938  * @param[in, out] matcher
4939  *   Flow matcher.
4940  * @param[in, out] key
4941  *   Flow matcher value.
4942  * @param[in] reg_type
4943  *   Type of device metadata register
4944  * @param[in] value
4945  *   Register value
4946  * @param[in] mask
4947  *   Register mask
4948  */
4949 static void
4950 flow_dv_match_meta_reg(void *matcher, void *key,
4951                        enum modify_reg reg_type,
4952                        uint32_t data, uint32_t mask)
4953 {
4954         void *misc2_m =
4955                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4956         void *misc2_v =
4957                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4958
4959         data &= mask;
4960         switch (reg_type) {
4961         case REG_A:
4962                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
4963                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
4964                 break;
4965         case REG_B:
4966                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
4967                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
4968                 break;
4969         case REG_C_0:
4970                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, mask);
4971                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, data);
4972                 break;
4973         case REG_C_1:
4974                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
4975                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
4976                 break;
4977         case REG_C_2:
4978                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
4979                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
4980                 break;
4981         case REG_C_3:
4982                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
4983                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
4984                 break;
4985         case REG_C_4:
4986                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
4987                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
4988                 break;
4989         case REG_C_5:
4990                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
4991                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
4992                 break;
4993         case REG_C_6:
4994                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
4995                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
4996                 break;
4997         case REG_C_7:
4998                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
4999                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
5000                 break;
5001         default:
5002                 assert(false);
5003                 break;
5004         }
5005 }
5006
5007 /**
5008  * Add META item to matcher
5009  *
5010  * @param[in, out] matcher
5011  *   Flow matcher.
5012  * @param[in, out] key
5013  *   Flow matcher value.
5014  * @param[in] item
5015  *   Flow pattern to translate.
5016  * @param[in] inner
5017  *   Item is inner pattern.
5018  */
5019 static void
5020 flow_dv_translate_item_meta(void *matcher, void *key,
5021                             const struct rte_flow_item *item)
5022 {
5023         const struct rte_flow_item_meta *meta_m;
5024         const struct rte_flow_item_meta *meta_v;
5025
5026         meta_m = (const void *)item->mask;
5027         if (!meta_m)
5028                 meta_m = &rte_flow_item_meta_mask;
5029         meta_v = (const void *)item->spec;
5030         if (meta_v)
5031                 flow_dv_match_meta_reg(matcher, key, REG_A,
5032                                        rte_cpu_to_be_32(meta_v->data),
5033                                        rte_cpu_to_be_32(meta_m->data));
5034 }
5035
5036 /**
5037  * Add vport metadata Reg C0 item to matcher
5038  *
5039  * @param[in, out] matcher
5040  *   Flow matcher.
5041  * @param[in, out] key
5042  *   Flow matcher value.
5043  * @param[in] reg
5044  *   Flow pattern to translate.
5045  */
5046 static void
5047 flow_dv_translate_item_meta_vport(void *matcher, void *key,
5048                                   uint32_t value, uint32_t mask)
5049 {
5050         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
5051 }
5052
5053 /**
5054  * Add tag item to matcher
5055  *
5056  * @param[in, out] matcher
5057  *   Flow matcher.
5058  * @param[in, out] key
5059  *   Flow matcher value.
5060  * @param[in] item
5061  *   Flow pattern to translate.
5062  */
5063 static void
5064 flow_dv_translate_mlx5_item_tag(void *matcher, void *key,
5065                                 const struct rte_flow_item *item)
5066 {
5067         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
5068         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
5069         enum modify_reg reg = tag_v->id;
5070
5071         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
5072 }
5073
5074 /**
5075  * Add source vport match to the specified matcher.
5076  *
5077  * @param[in, out] matcher
5078  *   Flow matcher.
5079  * @param[in, out] key
5080  *   Flow matcher value.
5081  * @param[in] port
5082  *   Source vport value to match
5083  * @param[in] mask
5084  *   Mask
5085  */
5086 static void
5087 flow_dv_translate_item_source_vport(void *matcher, void *key,
5088                                     int16_t port, uint16_t mask)
5089 {
5090         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5091         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5092
5093         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
5094         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
5095 }
5096
5097 /**
5098  * Translate port-id item to eswitch match on  port-id.
5099  *
5100  * @param[in] dev
5101  *   The devich to configure through.
5102  * @param[in, out] matcher
5103  *   Flow matcher.
5104  * @param[in, out] key
5105  *   Flow matcher value.
5106  * @param[in] item
5107  *   Flow pattern to translate.
5108  *
5109  * @return
5110  *   0 on success, a negative errno value otherwise.
5111  */
5112 static int
5113 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
5114                                void *key, const struct rte_flow_item *item)
5115 {
5116         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
5117         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
5118         struct mlx5_priv *priv;
5119         uint16_t mask, id;
5120
5121         mask = pid_m ? pid_m->id : 0xffff;
5122         id = pid_v ? pid_v->id : dev->data->port_id;
5123         priv = mlx5_port_to_eswitch_info(id, item == NULL);
5124         if (!priv)
5125                 return -rte_errno;
5126         /* Translate to vport field or to metadata, depending on mode. */
5127         if (priv->vport_meta_mask)
5128                 flow_dv_translate_item_meta_vport(matcher, key,
5129                                                   priv->vport_meta_tag,
5130                                                   priv->vport_meta_mask);
5131         else
5132                 flow_dv_translate_item_source_vport(matcher, key,
5133                                                     priv->vport_id, mask);
5134         return 0;
5135 }
5136
5137 /**
5138  * Add ICMP6 item to matcher and to the value.
5139  *
5140  * @param[in, out] matcher
5141  *   Flow matcher.
5142  * @param[in, out] key
5143  *   Flow matcher value.
5144  * @param[in] item
5145  *   Flow pattern to translate.
5146  * @param[in] inner
5147  *   Item is inner pattern.
5148  */
5149 static void
5150 flow_dv_translate_item_icmp6(void *matcher, void *key,
5151                               const struct rte_flow_item *item,
5152                               int inner)
5153 {
5154         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
5155         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
5156         void *headers_m;
5157         void *headers_v;
5158         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
5159                                      misc_parameters_3);
5160         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
5161         if (inner) {
5162                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5163                                          inner_headers);
5164                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5165         } else {
5166                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5167                                          outer_headers);
5168                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5169         }
5170         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
5171         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
5172         if (!icmp6_v)
5173                 return;
5174         if (!icmp6_m)
5175                 icmp6_m = &rte_flow_item_icmp6_mask;
5176         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
5177         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
5178                  icmp6_v->type & icmp6_m->type);
5179         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
5180         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
5181                  icmp6_v->code & icmp6_m->code);
5182 }
5183
5184 /**
5185  * Add ICMP item to matcher and to the value.
5186  *
5187  * @param[in, out] matcher
5188  *   Flow matcher.
5189  * @param[in, out] key
5190  *   Flow matcher value.
5191  * @param[in] item
5192  *   Flow pattern to translate.
5193  * @param[in] inner
5194  *   Item is inner pattern.
5195  */
5196 static void
5197 flow_dv_translate_item_icmp(void *matcher, void *key,
5198                             const struct rte_flow_item *item,
5199                             int inner)
5200 {
5201         const struct rte_flow_item_icmp *icmp_m = item->mask;
5202         const struct rte_flow_item_icmp *icmp_v = item->spec;
5203         void *headers_m;
5204         void *headers_v;
5205         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
5206                                      misc_parameters_3);
5207         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
5208         if (inner) {
5209                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5210                                          inner_headers);
5211                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5212         } else {
5213                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5214                                          outer_headers);
5215                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5216         }
5217         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
5218         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
5219         if (!icmp_v)
5220                 return;
5221         if (!icmp_m)
5222                 icmp_m = &rte_flow_item_icmp_mask;
5223         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
5224                  icmp_m->hdr.icmp_type);
5225         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
5226                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
5227         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
5228                  icmp_m->hdr.icmp_code);
5229         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
5230                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
5231 }
5232
5233 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
5234
5235 #define HEADER_IS_ZERO(match_criteria, headers)                              \
5236         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
5237                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
5238
5239 /**
5240  * Calculate flow matcher enable bitmap.
5241  *
5242  * @param match_criteria
5243  *   Pointer to flow matcher criteria.
5244  *
5245  * @return
5246  *   Bitmap of enabled fields.
5247  */
5248 static uint8_t
5249 flow_dv_matcher_enable(uint32_t *match_criteria)
5250 {
5251         uint8_t match_criteria_enable;
5252
5253         match_criteria_enable =
5254                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
5255                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
5256         match_criteria_enable |=
5257                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
5258                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
5259         match_criteria_enable |=
5260                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
5261                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
5262         match_criteria_enable |=
5263                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
5264                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
5265         match_criteria_enable |=
5266                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
5267                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
5268         return match_criteria_enable;
5269 }
5270
5271
5272 /**
5273  * Get a flow table.
5274  *
5275  * @param dev[in, out]
5276  *   Pointer to rte_eth_dev structure.
5277  * @param[in] table_id
5278  *   Table id to use.
5279  * @param[in] egress
5280  *   Direction of the table.
5281  * @param[in] transfer
5282  *   E-Switch or NIC flow.
5283  * @param[out] error
5284  *   pointer to error structure.
5285  *
5286  * @return
5287  *   Returns tables resource based on the index, NULL in case of failed.
5288  */
5289 static struct mlx5_flow_tbl_resource *
5290 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
5291                          uint32_t table_id, uint8_t egress,
5292                          uint8_t transfer,
5293                          struct rte_flow_error *error)
5294 {
5295         struct mlx5_priv *priv = dev->data->dev_private;
5296         struct mlx5_ibv_shared *sh = priv->sh;
5297         struct mlx5_flow_tbl_resource *tbl;
5298
5299 #ifdef HAVE_MLX5DV_DR
5300         if (transfer) {
5301                 tbl = &sh->fdb_tbl[table_id];
5302                 if (!tbl->obj)
5303                         tbl->obj = mlx5_glue->dr_create_flow_tbl
5304                                 (sh->fdb_domain, table_id);
5305         } else if (egress) {
5306                 tbl = &sh->tx_tbl[table_id];
5307                 if (!tbl->obj)
5308                         tbl->obj = mlx5_glue->dr_create_flow_tbl
5309                                 (sh->tx_domain, table_id);
5310         } else {
5311                 tbl = &sh->rx_tbl[table_id];
5312                 if (!tbl->obj)
5313                         tbl->obj = mlx5_glue->dr_create_flow_tbl
5314                                 (sh->rx_domain, table_id);
5315         }
5316         if (!tbl->obj) {
5317                 rte_flow_error_set(error, ENOMEM,
5318                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5319                                    NULL, "cannot create table");
5320                 return NULL;
5321         }
5322         rte_atomic32_inc(&tbl->refcnt);
5323         return tbl;
5324 #else
5325         (void)error;
5326         (void)tbl;
5327         if (transfer)
5328                 return &sh->fdb_tbl[table_id];
5329         else if (egress)
5330                 return &sh->tx_tbl[table_id];
5331         else
5332                 return &sh->rx_tbl[table_id];
5333 #endif
5334 }
5335
5336 /**
5337  * Release a flow table.
5338  *
5339  * @param[in] tbl
5340  *   Table resource to be released.
5341  *
5342  * @return
5343  *   Returns 0 if table was released, else return 1;
5344  */
5345 static int
5346 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
5347 {
5348         if (!tbl)
5349                 return 0;
5350         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
5351                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
5352                 tbl->obj = NULL;
5353                 return 0;
5354         }
5355         return 1;
5356 }
5357
5358 /**
5359  * Register the flow matcher.
5360  *
5361  * @param dev[in, out]
5362  *   Pointer to rte_eth_dev structure.
5363  * @param[in, out] matcher
5364  *   Pointer to flow matcher.
5365  * @parm[in, out] dev_flow
5366  *   Pointer to the dev_flow.
5367  * @param[out] error
5368  *   pointer to error structure.
5369  *
5370  * @return
5371  *   0 on success otherwise -errno and errno is set.
5372  */
5373 static int
5374 flow_dv_matcher_register(struct rte_eth_dev *dev,
5375                          struct mlx5_flow_dv_matcher *matcher,
5376                          struct mlx5_flow *dev_flow,
5377                          struct rte_flow_error *error)
5378 {
5379         struct mlx5_priv *priv = dev->data->dev_private;
5380         struct mlx5_ibv_shared *sh = priv->sh;
5381         struct mlx5_flow_dv_matcher *cache_matcher;
5382         struct mlx5dv_flow_matcher_attr dv_attr = {
5383                 .type = IBV_FLOW_ATTR_NORMAL,
5384                 .match_mask = (void *)&matcher->mask,
5385         };
5386         struct mlx5_flow_tbl_resource *tbl = NULL;
5387
5388         /* Lookup from cache. */
5389         LIST_FOREACH(cache_matcher, &sh->matchers, next) {
5390                 if (matcher->crc == cache_matcher->crc &&
5391                     matcher->priority == cache_matcher->priority &&
5392                     matcher->egress == cache_matcher->egress &&
5393                     matcher->group == cache_matcher->group &&
5394                     matcher->transfer == cache_matcher->transfer &&
5395                     !memcmp((const void *)matcher->mask.buf,
5396                             (const void *)cache_matcher->mask.buf,
5397                             cache_matcher->mask.size)) {
5398                         DRV_LOG(DEBUG,
5399                                 "priority %hd use %s matcher %p: refcnt %d++",
5400                                 cache_matcher->priority,
5401                                 cache_matcher->egress ? "tx" : "rx",
5402                                 (void *)cache_matcher,
5403                                 rte_atomic32_read(&cache_matcher->refcnt));
5404                         rte_atomic32_inc(&cache_matcher->refcnt);
5405                         dev_flow->dv.matcher = cache_matcher;
5406                         return 0;
5407                 }
5408         }
5409         /* Register new matcher. */
5410         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
5411         if (!cache_matcher)
5412                 return rte_flow_error_set(error, ENOMEM,
5413                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5414                                           "cannot allocate matcher memory");
5415         tbl = flow_dv_tbl_resource_get(dev, matcher->group,
5416                                        matcher->egress, matcher->transfer,
5417                                        error);
5418         if (!tbl) {
5419                 rte_free(cache_matcher);
5420                 return rte_flow_error_set(error, ENOMEM,
5421                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5422                                           NULL, "cannot create table");
5423         }
5424         *cache_matcher = *matcher;
5425         dv_attr.match_criteria_enable =
5426                 flow_dv_matcher_enable(cache_matcher->mask.buf);
5427         dv_attr.priority = matcher->priority;
5428         if (matcher->egress)
5429                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
5430         cache_matcher->matcher_object =
5431                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
5432         if (!cache_matcher->matcher_object) {
5433                 rte_free(cache_matcher);
5434 #ifdef HAVE_MLX5DV_DR
5435                 flow_dv_tbl_resource_release(tbl);
5436 #endif
5437                 return rte_flow_error_set(error, ENOMEM,
5438                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5439                                           NULL, "cannot create matcher");
5440         }
5441         rte_atomic32_inc(&cache_matcher->refcnt);
5442         LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
5443         dev_flow->dv.matcher = cache_matcher;
5444         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
5445                 cache_matcher->priority,
5446                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
5447                 rte_atomic32_read(&cache_matcher->refcnt));
5448         rte_atomic32_inc(&tbl->refcnt);
5449         return 0;
5450 }
5451
5452 /**
5453  * Find existing tag resource or create and register a new one.
5454  *
5455  * @param dev[in, out]
5456  *   Pointer to rte_eth_dev structure.
5457  * @param[in, out] resource
5458  *   Pointer to tag resource.
5459  * @parm[in, out] dev_flow
5460  *   Pointer to the dev_flow.
5461  * @param[out] error
5462  *   pointer to error structure.
5463  *
5464  * @return
5465  *   0 on success otherwise -errno and errno is set.
5466  */
5467 static int
5468 flow_dv_tag_resource_register
5469                         (struct rte_eth_dev *dev,
5470                          struct mlx5_flow_dv_tag_resource *resource,
5471                          struct mlx5_flow *dev_flow,
5472                          struct rte_flow_error *error)
5473 {
5474         struct mlx5_priv *priv = dev->data->dev_private;
5475         struct mlx5_ibv_shared *sh = priv->sh;
5476         struct mlx5_flow_dv_tag_resource *cache_resource;
5477
5478         /* Lookup a matching resource from cache. */
5479         LIST_FOREACH(cache_resource, &sh->tags, next) {
5480                 if (resource->tag == cache_resource->tag) {
5481                         DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
5482                                 (void *)cache_resource,
5483                                 rte_atomic32_read(&cache_resource->refcnt));
5484                         rte_atomic32_inc(&cache_resource->refcnt);
5485                         dev_flow->dv.tag_resource = cache_resource;
5486                         return 0;
5487                 }
5488         }
5489         /* Register new  resource. */
5490         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
5491         if (!cache_resource)
5492                 return rte_flow_error_set(error, ENOMEM,
5493                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5494                                           "cannot allocate resource memory");
5495         *cache_resource = *resource;
5496         cache_resource->action = mlx5_glue->dv_create_flow_action_tag
5497                 (resource->tag);
5498         if (!cache_resource->action) {
5499                 rte_free(cache_resource);
5500                 return rte_flow_error_set(error, ENOMEM,
5501                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5502                                           NULL, "cannot create action");
5503         }
5504         rte_atomic32_init(&cache_resource->refcnt);
5505         rte_atomic32_inc(&cache_resource->refcnt);
5506         LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
5507         dev_flow->dv.tag_resource = cache_resource;
5508         DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
5509                 (void *)cache_resource,
5510                 rte_atomic32_read(&cache_resource->refcnt));
5511         return 0;
5512 }
5513
5514 /**
5515  * Release the tag.
5516  *
5517  * @param dev
5518  *   Pointer to Ethernet device.
5519  * @param flow
5520  *   Pointer to mlx5_flow.
5521  *
5522  * @return
5523  *   1 while a reference on it exists, 0 when freed.
5524  */
5525 static int
5526 flow_dv_tag_release(struct rte_eth_dev *dev,
5527                     struct mlx5_flow_dv_tag_resource *tag)
5528 {
5529         assert(tag);
5530         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
5531                 dev->data->port_id, (void *)tag,
5532                 rte_atomic32_read(&tag->refcnt));
5533         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
5534                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
5535                 LIST_REMOVE(tag, next);
5536                 DRV_LOG(DEBUG, "port %u tag %p: removed",
5537                         dev->data->port_id, (void *)tag);
5538                 rte_free(tag);
5539                 return 0;
5540         }
5541         return 1;
5542 }
5543
5544 /**
5545  * Translate port ID action to vport.
5546  *
5547  * @param[in] dev
5548  *   Pointer to rte_eth_dev structure.
5549  * @param[in] action
5550  *   Pointer to the port ID action.
5551  * @param[out] dst_port_id
5552  *   The target port ID.
5553  * @param[out] error
5554  *   Pointer to the error structure.
5555  *
5556  * @return
5557  *   0 on success, a negative errno value otherwise and rte_errno is set.
5558  */
5559 static int
5560 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
5561                                  const struct rte_flow_action *action,
5562                                  uint32_t *dst_port_id,
5563                                  struct rte_flow_error *error)
5564 {
5565         uint32_t port;
5566         struct mlx5_priv *priv;
5567         const struct rte_flow_action_port_id *conf =
5568                         (const struct rte_flow_action_port_id *)action->conf;
5569
5570         port = conf->original ? dev->data->port_id : conf->id;
5571         priv = mlx5_port_to_eswitch_info(port, false);
5572         if (!priv)
5573                 return rte_flow_error_set(error, -rte_errno,
5574                                           RTE_FLOW_ERROR_TYPE_ACTION,
5575                                           NULL,
5576                                           "No eswitch info was found for port");
5577         if (priv->vport_meta_mask)
5578                 *dst_port_id = priv->vport_meta_tag;
5579         else
5580                 *dst_port_id = priv->vport_id;
5581         return 0;
5582 }
5583
5584 /**
5585  * Add Tx queue matcher
5586  *
5587  * @param[in] dev
5588  *   Pointer to the dev struct.
5589  * @param[in, out] matcher
5590  *   Flow matcher.
5591  * @param[in, out] key
5592  *   Flow matcher value.
5593  * @param[in] item
5594  *   Flow pattern to translate.
5595  * @param[in] inner
5596  *   Item is inner pattern.
5597  */
5598 static void
5599 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
5600                                 void *matcher, void *key,
5601                                 const struct rte_flow_item *item)
5602 {
5603         const struct mlx5_rte_flow_item_tx_queue *queue_m;
5604         const struct mlx5_rte_flow_item_tx_queue *queue_v;
5605         void *misc_m =
5606                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
5607         void *misc_v =
5608                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
5609         struct mlx5_txq_ctrl *txq;
5610         uint32_t queue;
5611
5612
5613         queue_m = (const void *)item->mask;
5614         if (!queue_m)
5615                 return;
5616         queue_v = (const void *)item->spec;
5617         if (!queue_v)
5618                 return;
5619         txq = mlx5_txq_get(dev, queue_v->queue);
5620         if (!txq)
5621                 return;
5622         queue = txq->obj->sq->id;
5623         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
5624         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
5625                  queue & queue_m->queue);
5626         mlx5_txq_release(dev, queue_v->queue);
5627 }
5628
5629 /**
5630  * Fill the flow with DV spec, lock free
5631  * (mutex should be acquired by caller).
5632  *
5633  * @param[in] dev
5634  *   Pointer to rte_eth_dev structure.
5635  * @param[in, out] dev_flow
5636  *   Pointer to the sub flow.
5637  * @param[in] attr
5638  *   Pointer to the flow attributes.
5639  * @param[in] items
5640  *   Pointer to the list of items.
5641  * @param[in] actions
5642  *   Pointer to the list of actions.
5643  * @param[out] error
5644  *   Pointer to the error structure.
5645  *
5646  * @return
5647  *   0 on success, a negative errno value otherwise and rte_errno is set.
5648  */
5649 static int
5650 __flow_dv_translate(struct rte_eth_dev *dev,
5651                     struct mlx5_flow *dev_flow,
5652                     const struct rte_flow_attr *attr,
5653                     const struct rte_flow_item items[],
5654                     const struct rte_flow_action actions[],
5655                     struct rte_flow_error *error)
5656 {
5657         struct mlx5_priv *priv = dev->data->dev_private;
5658         struct rte_flow *flow = dev_flow->flow;
5659         uint64_t item_flags = 0;
5660         uint64_t last_item = 0;
5661         uint64_t action_flags = 0;
5662         uint64_t priority = attr->priority;
5663         struct mlx5_flow_dv_matcher matcher = {
5664                 .mask = {
5665                         .size = sizeof(matcher.mask.buf),
5666                 },
5667         };
5668         int actions_n = 0;
5669         bool actions_end = false;
5670         struct mlx5_flow_dv_modify_hdr_resource mhdr_res = {
5671                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
5672                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
5673         };
5674         union flow_dv_attr flow_attr = { .attr = 0 };
5675         struct mlx5_flow_dv_tag_resource tag_resource;
5676         uint32_t modify_action_position = UINT32_MAX;
5677         void *match_mask = matcher.mask.buf;
5678         void *match_value = dev_flow->dv.value.buf;
5679         uint8_t next_protocol = 0xff;
5680         struct rte_vlan_hdr vlan = { 0 };
5681         uint32_t table;
5682         int ret = 0;
5683
5684         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
5685                                        &table, error);
5686         if (ret)
5687                 return ret;
5688         dev_flow->group = table;
5689         if (attr->transfer)
5690                 mhdr_res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
5691         if (priority == MLX5_FLOW_PRIO_RSVD)
5692                 priority = priv->config.flow_prio - 1;
5693         for (; !actions_end ; actions++) {
5694                 const struct rte_flow_action_queue *queue;
5695                 const struct rte_flow_action_rss *rss;
5696                 const struct rte_flow_action *action = actions;
5697                 const struct rte_flow_action_count *count = action->conf;
5698                 const uint8_t *rss_key;
5699                 const struct rte_flow_action_jump *jump_data;
5700                 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
5701                 struct mlx5_flow_tbl_resource *tbl;
5702                 uint32_t port_id = 0;
5703                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
5704                 int action_type = actions->type;
5705                 const struct rte_flow_action *found_action = NULL;
5706
5707                 switch (action_type) {
5708                 case RTE_FLOW_ACTION_TYPE_VOID:
5709                         break;
5710                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5711                         if (flow_dv_translate_action_port_id(dev, action,
5712                                                              &port_id, error))
5713                                 return -rte_errno;
5714                         port_id_resource.port_id = port_id;
5715                         if (flow_dv_port_id_action_resource_register
5716                             (dev, &port_id_resource, dev_flow, error))
5717                                 return -rte_errno;
5718                         dev_flow->dv.actions[actions_n++] =
5719                                 dev_flow->dv.port_id_action->action;
5720                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5721                         break;
5722                 case RTE_FLOW_ACTION_TYPE_FLAG:
5723                         tag_resource.tag =
5724                                 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
5725                         if (!dev_flow->dv.tag_resource)
5726                                 if (flow_dv_tag_resource_register
5727                                     (dev, &tag_resource, dev_flow, error))
5728                                         return errno;
5729                         dev_flow->dv.actions[actions_n++] =
5730                                 dev_flow->dv.tag_resource->action;
5731                         action_flags |= MLX5_FLOW_ACTION_FLAG;
5732                         break;
5733                 case RTE_FLOW_ACTION_TYPE_MARK:
5734                         tag_resource.tag = mlx5_flow_mark_set
5735                               (((const struct rte_flow_action_mark *)
5736                                (actions->conf))->id);
5737                         if (!dev_flow->dv.tag_resource)
5738                                 if (flow_dv_tag_resource_register
5739                                     (dev, &tag_resource, dev_flow, error))
5740                                         return errno;
5741                         dev_flow->dv.actions[actions_n++] =
5742                                 dev_flow->dv.tag_resource->action;
5743                         action_flags |= MLX5_FLOW_ACTION_MARK;
5744                         break;
5745                 case RTE_FLOW_ACTION_TYPE_DROP:
5746                         action_flags |= MLX5_FLOW_ACTION_DROP;
5747                         break;
5748                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5749                         assert(flow->rss.queue);
5750                         queue = actions->conf;
5751                         flow->rss.queue_num = 1;
5752                         (*flow->rss.queue)[0] = queue->index;
5753                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5754                         break;
5755                 case RTE_FLOW_ACTION_TYPE_RSS:
5756                         assert(flow->rss.queue);
5757                         rss = actions->conf;
5758                         if (flow->rss.queue)
5759                                 memcpy((*flow->rss.queue), rss->queue,
5760                                        rss->queue_num * sizeof(uint16_t));
5761                         flow->rss.queue_num = rss->queue_num;
5762                         /* NULL RSS key indicates default RSS key. */
5763                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
5764                         memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
5765                         /*
5766                          * rss->level and rss.types should be set in advance
5767                          * when expanding items for RSS.
5768                          */
5769                         action_flags |= MLX5_FLOW_ACTION_RSS;
5770                         break;
5771                 case RTE_FLOW_ACTION_TYPE_COUNT:
5772                         if (!priv->config.devx) {
5773                                 rte_errno = ENOTSUP;
5774                                 goto cnt_err;
5775                         }
5776                         flow->counter = flow_dv_counter_alloc(dev,
5777                                                               count->shared,
5778                                                               count->id,
5779                                                               dev_flow->group);
5780                         if (flow->counter == NULL)
5781                                 goto cnt_err;
5782                         dev_flow->dv.actions[actions_n++] =
5783                                 flow->counter->action;
5784                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5785                         break;
5786 cnt_err:
5787                         if (rte_errno == ENOTSUP)
5788                                 return rte_flow_error_set
5789                                               (error, ENOTSUP,
5790                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5791                                                NULL,
5792                                                "count action not supported");
5793                         else
5794                                 return rte_flow_error_set
5795                                                 (error, rte_errno,
5796                                                  RTE_FLOW_ERROR_TYPE_ACTION,
5797                                                  action,
5798                                                  "cannot create counter"
5799                                                   " object.");
5800                         break;
5801                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5802                         dev_flow->dv.actions[actions_n++] =
5803                                                 priv->sh->pop_vlan_action;
5804                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5805                         break;
5806                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5807                         flow_dev_get_vlan_info_from_items(items, &vlan);
5808                         vlan.eth_proto = rte_be_to_cpu_16
5809                              ((((const struct rte_flow_action_of_push_vlan *)
5810                                                    actions->conf)->ethertype));
5811                         found_action = mlx5_flow_find_action
5812                                         (actions + 1,
5813                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
5814                         if (found_action)
5815                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
5816                         found_action = mlx5_flow_find_action
5817                                         (actions + 1,
5818                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
5819                         if (found_action)
5820                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
5821                         if (flow_dv_create_action_push_vlan
5822                                             (dev, attr, &vlan, dev_flow, error))
5823                                 return -rte_errno;
5824                         dev_flow->dv.actions[actions_n++] =
5825                                            dev_flow->dv.push_vlan_res->action;
5826                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5827                         break;
5828                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5829                         /* of_vlan_push action handled this action */
5830                         assert(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN);
5831                         break;
5832                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5833                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
5834                                 break;
5835                         flow_dev_get_vlan_info_from_items(items, &vlan);
5836                         mlx5_update_vlan_vid_pcp(actions, &vlan);
5837                         /* If no VLAN push - this is a modify header action */
5838                         if (flow_dv_convert_action_modify_vlan_vid
5839                                                 (&mhdr_res, actions, error))
5840                                 return -rte_errno;
5841                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5842                         break;
5843                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5844                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5845                         if (flow_dv_create_action_l2_encap(dev, actions,
5846                                                            dev_flow,
5847                                                            attr->transfer,
5848                                                            error))
5849                                 return -rte_errno;
5850                         dev_flow->dv.actions[actions_n++] =
5851                                 dev_flow->dv.encap_decap->verbs_action;
5852                         action_flags |= actions->type ==
5853                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
5854                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
5855                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
5856                         break;
5857                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5858                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5859                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
5860                                                            attr->transfer,
5861                                                            error))
5862                                 return -rte_errno;
5863                         dev_flow->dv.actions[actions_n++] =
5864                                 dev_flow->dv.encap_decap->verbs_action;
5865                         action_flags |= actions->type ==
5866                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
5867                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
5868                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
5869                         break;
5870                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5871                         /* Handle encap with preceding decap. */
5872                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
5873                                 if (flow_dv_create_action_raw_encap
5874                                         (dev, actions, dev_flow, attr, error))
5875                                         return -rte_errno;
5876                                 dev_flow->dv.actions[actions_n++] =
5877                                         dev_flow->dv.encap_decap->verbs_action;
5878                         } else {
5879                                 /* Handle encap without preceding decap. */
5880                                 if (flow_dv_create_action_l2_encap
5881                                     (dev, actions, dev_flow, attr->transfer,
5882                                      error))
5883                                         return -rte_errno;
5884                                 dev_flow->dv.actions[actions_n++] =
5885                                         dev_flow->dv.encap_decap->verbs_action;
5886                         }
5887                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
5888                         break;
5889                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5890                         /* Check if this decap is followed by encap. */
5891                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
5892                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
5893                                action++) {
5894                         }
5895                         /* Handle decap only if it isn't followed by encap. */
5896                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5897                                 if (flow_dv_create_action_l2_decap
5898                                     (dev, dev_flow, attr->transfer, error))
5899                                         return -rte_errno;
5900                                 dev_flow->dv.actions[actions_n++] =
5901                                         dev_flow->dv.encap_decap->verbs_action;
5902                         }
5903                         /* If decap is followed by encap, handle it at encap. */
5904                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
5905                         break;
5906                 case RTE_FLOW_ACTION_TYPE_JUMP:
5907                         jump_data = action->conf;
5908                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
5909                                                        jump_data->group, &table,
5910                                                        error);
5911                         if (ret)
5912                                 return ret;
5913                         tbl = flow_dv_tbl_resource_get(dev, table,
5914                                                        attr->egress,
5915                                                        attr->transfer, error);
5916                         if (!tbl)
5917                                 return rte_flow_error_set
5918                                                 (error, errno,
5919                                                  RTE_FLOW_ERROR_TYPE_ACTION,
5920                                                  NULL,
5921                                                  "cannot create jump action.");
5922                         jump_tbl_resource.tbl = tbl;
5923                         if (flow_dv_jump_tbl_resource_register
5924                             (dev, &jump_tbl_resource, dev_flow, error)) {
5925                                 flow_dv_tbl_resource_release(tbl);
5926                                 return rte_flow_error_set
5927                                                 (error, errno,
5928                                                  RTE_FLOW_ERROR_TYPE_ACTION,
5929                                                  NULL,
5930                                                  "cannot create jump action.");
5931                         }
5932                         dev_flow->dv.actions[actions_n++] =
5933                                 dev_flow->dv.jump->action;
5934                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5935                         break;
5936                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5937                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5938                         if (flow_dv_convert_action_modify_mac
5939                                         (&mhdr_res, actions, error))
5940                                 return -rte_errno;
5941                         action_flags |= actions->type ==
5942                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5943                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
5944                                         MLX5_FLOW_ACTION_SET_MAC_DST;
5945                         break;
5946                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5947                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5948                         if (flow_dv_convert_action_modify_ipv4
5949                                         (&mhdr_res, actions, error))
5950                                 return -rte_errno;
5951                         action_flags |= actions->type ==
5952                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5953                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
5954                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
5955                         break;
5956                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5957                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5958                         if (flow_dv_convert_action_modify_ipv6
5959                                         (&mhdr_res, actions, error))
5960                                 return -rte_errno;
5961                         action_flags |= actions->type ==
5962                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5963                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
5964                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
5965                         break;
5966                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5967                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5968                         if (flow_dv_convert_action_modify_tp
5969                                         (&mhdr_res, actions, items,
5970                                          &flow_attr, error))
5971                                 return -rte_errno;
5972                         action_flags |= actions->type ==
5973                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5974                                         MLX5_FLOW_ACTION_SET_TP_SRC :
5975                                         MLX5_FLOW_ACTION_SET_TP_DST;
5976                         break;
5977                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5978                         if (flow_dv_convert_action_modify_dec_ttl
5979                                         (&mhdr_res, items, &flow_attr, error))
5980                                 return -rte_errno;
5981                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
5982                         break;
5983                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5984                         if (flow_dv_convert_action_modify_ttl
5985                                         (&mhdr_res, actions, items,
5986                                          &flow_attr, error))
5987                                 return -rte_errno;
5988                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
5989                         break;
5990                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5991                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5992                         if (flow_dv_convert_action_modify_tcp_seq
5993                                         (&mhdr_res, actions, error))
5994                                 return -rte_errno;
5995                         action_flags |= actions->type ==
5996                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5997                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
5998                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5999                         break;
6000
6001                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6002                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6003                         if (flow_dv_convert_action_modify_tcp_ack
6004                                         (&mhdr_res, actions, error))
6005                                 return -rte_errno;
6006                         action_flags |= actions->type ==
6007                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
6008                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
6009                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
6010                         break;
6011                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
6012                         if (flow_dv_convert_action_set_reg
6013                                         (&mhdr_res, actions, error))
6014                                 return -rte_errno;
6015                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
6016                         break;
6017                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
6018                         if (flow_dv_convert_action_copy_mreg
6019                                         (dev, &mhdr_res, actions, error))
6020                                 return -rte_errno;
6021                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
6022                         break;
6023                 case RTE_FLOW_ACTION_TYPE_END:
6024                         actions_end = true;
6025                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
6026                                 /* create modify action if needed. */
6027                                 if (flow_dv_modify_hdr_resource_register
6028                                         (dev, &mhdr_res, dev_flow, error))
6029                                         return -rte_errno;
6030                                 dev_flow->dv.actions[modify_action_position] =
6031                                         dev_flow->dv.modify_hdr->verbs_action;
6032                         }
6033                         break;
6034                 default:
6035                         break;
6036                 }
6037                 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
6038                     modify_action_position == UINT32_MAX)
6039                         modify_action_position = actions_n++;
6040         }
6041         dev_flow->dv.actions_n = actions_n;
6042         dev_flow->actions = action_flags;
6043         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6044                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6045                 int item_type = items->type;
6046
6047                 switch (item_type) {
6048                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6049                         flow_dv_translate_item_port_id(dev, match_mask,
6050                                                        match_value, items);
6051                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6052                         break;
6053                 case RTE_FLOW_ITEM_TYPE_ETH:
6054                         flow_dv_translate_item_eth(match_mask, match_value,
6055                                                    items, tunnel);
6056                         matcher.priority = MLX5_PRIORITY_MAP_L2;
6057                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6058                                              MLX5_FLOW_LAYER_OUTER_L2;
6059                         break;
6060                 case RTE_FLOW_ITEM_TYPE_VLAN:
6061                         flow_dv_translate_item_vlan(dev_flow,
6062                                                     match_mask, match_value,
6063                                                     items, tunnel);
6064                         matcher.priority = MLX5_PRIORITY_MAP_L2;
6065                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
6066                                               MLX5_FLOW_LAYER_INNER_VLAN) :
6067                                              (MLX5_FLOW_LAYER_OUTER_L2 |
6068                                               MLX5_FLOW_LAYER_OUTER_VLAN);
6069                         break;
6070                 case RTE_FLOW_ITEM_TYPE_IPV4:
6071                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6072                                                   &item_flags, &tunnel);
6073                         flow_dv_translate_item_ipv4(match_mask, match_value,
6074                                                     items, tunnel,
6075                                                     dev_flow->group);
6076                         matcher.priority = MLX5_PRIORITY_MAP_L3;
6077                         dev_flow->hash_fields |=
6078                                 mlx5_flow_hashfields_adjust
6079                                         (dev_flow, tunnel,
6080                                          MLX5_IPV4_LAYER_TYPES,
6081                                          MLX5_IPV4_IBV_RX_HASH);
6082                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
6083                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
6084                         if (items->mask != NULL &&
6085                             ((const struct rte_flow_item_ipv4 *)
6086                              items->mask)->hdr.next_proto_id) {
6087                                 next_protocol =
6088                                         ((const struct rte_flow_item_ipv4 *)
6089                                          (items->spec))->hdr.next_proto_id;
6090                                 next_protocol &=
6091                                         ((const struct rte_flow_item_ipv4 *)
6092                                          (items->mask))->hdr.next_proto_id;
6093                         } else {
6094                                 /* Reset for inner layer. */
6095                                 next_protocol = 0xff;
6096                         }
6097                         break;
6098                 case RTE_FLOW_ITEM_TYPE_IPV6:
6099                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6100                                                   &item_flags, &tunnel);
6101                         flow_dv_translate_item_ipv6(match_mask, match_value,
6102                                                     items, tunnel,
6103                                                     dev_flow->group);
6104                         matcher.priority = MLX5_PRIORITY_MAP_L3;
6105                         dev_flow->hash_fields |=
6106                                 mlx5_flow_hashfields_adjust
6107                                         (dev_flow, tunnel,
6108                                          MLX5_IPV6_LAYER_TYPES,
6109                                          MLX5_IPV6_IBV_RX_HASH);
6110                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
6111                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
6112                         if (items->mask != NULL &&
6113                             ((const struct rte_flow_item_ipv6 *)
6114                              items->mask)->hdr.proto) {
6115                                 next_protocol =
6116                                         ((const struct rte_flow_item_ipv6 *)
6117                                          items->spec)->hdr.proto;
6118                                 next_protocol &=
6119                                         ((const struct rte_flow_item_ipv6 *)
6120                                          items->mask)->hdr.proto;
6121                         } else {
6122                                 /* Reset for inner layer. */
6123                                 next_protocol = 0xff;
6124                         }
6125                         break;
6126                 case RTE_FLOW_ITEM_TYPE_TCP:
6127                         flow_dv_translate_item_tcp(match_mask, match_value,
6128                                                    items, tunnel);
6129                         matcher.priority = MLX5_PRIORITY_MAP_L4;
6130                         dev_flow->hash_fields |=
6131                                 mlx5_flow_hashfields_adjust
6132                                         (dev_flow, tunnel, ETH_RSS_TCP,
6133                                          IBV_RX_HASH_SRC_PORT_TCP |
6134                                          IBV_RX_HASH_DST_PORT_TCP);
6135                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
6136                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
6137                         break;
6138                 case RTE_FLOW_ITEM_TYPE_UDP:
6139                         flow_dv_translate_item_udp(match_mask, match_value,
6140                                                    items, tunnel);
6141                         matcher.priority = MLX5_PRIORITY_MAP_L4;
6142                         dev_flow->hash_fields |=
6143                                 mlx5_flow_hashfields_adjust
6144                                         (dev_flow, tunnel, ETH_RSS_UDP,
6145                                          IBV_RX_HASH_SRC_PORT_UDP |
6146                                          IBV_RX_HASH_DST_PORT_UDP);
6147                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
6148                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
6149                         break;
6150                 case RTE_FLOW_ITEM_TYPE_GRE:
6151                         flow_dv_translate_item_gre(match_mask, match_value,
6152                                                    items, tunnel);
6153                         last_item = MLX5_FLOW_LAYER_GRE;
6154                         break;
6155                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
6156                         flow_dv_translate_item_gre_key(match_mask,
6157                                                        match_value, items);
6158                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
6159                         break;
6160                 case RTE_FLOW_ITEM_TYPE_NVGRE:
6161                         flow_dv_translate_item_nvgre(match_mask, match_value,
6162                                                      items, tunnel);
6163                         last_item = MLX5_FLOW_LAYER_GRE;
6164                         break;
6165                 case RTE_FLOW_ITEM_TYPE_VXLAN:
6166                         flow_dv_translate_item_vxlan(match_mask, match_value,
6167                                                      items, tunnel);
6168                         last_item = MLX5_FLOW_LAYER_VXLAN;
6169                         break;
6170                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
6171                         flow_dv_translate_item_vxlan(match_mask, match_value,
6172                                                      items, tunnel);
6173                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
6174                         break;
6175                 case RTE_FLOW_ITEM_TYPE_GENEVE:
6176                         flow_dv_translate_item_geneve(match_mask, match_value,
6177                                                       items, tunnel);
6178                         last_item = MLX5_FLOW_LAYER_GENEVE;
6179                         break;
6180                 case RTE_FLOW_ITEM_TYPE_MPLS:
6181                         flow_dv_translate_item_mpls(match_mask, match_value,
6182                                                     items, last_item, tunnel);
6183                         last_item = MLX5_FLOW_LAYER_MPLS;
6184                         break;
6185                 case RTE_FLOW_ITEM_TYPE_META:
6186                         flow_dv_translate_item_meta(match_mask, match_value,
6187                                                     items);
6188                         last_item = MLX5_FLOW_ITEM_METADATA;
6189                         break;
6190                 case RTE_FLOW_ITEM_TYPE_ICMP:
6191                         flow_dv_translate_item_icmp(match_mask, match_value,
6192                                                     items, tunnel);
6193                         last_item = MLX5_FLOW_LAYER_ICMP;
6194                         break;
6195                 case RTE_FLOW_ITEM_TYPE_ICMP6:
6196                         flow_dv_translate_item_icmp6(match_mask, match_value,
6197                                                       items, tunnel);
6198                         last_item = MLX5_FLOW_LAYER_ICMP6;
6199                         break;
6200                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
6201                         flow_dv_translate_mlx5_item_tag(match_mask,
6202                                                         match_value, items);
6203                         last_item = MLX5_FLOW_ITEM_TAG;
6204                         break;
6205                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
6206                         flow_dv_translate_item_tx_queue(dev, match_mask,
6207                                                         match_value,
6208                                                         items);
6209                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
6210                         break;
6211                 default:
6212                         break;
6213                 }
6214                 item_flags |= last_item;
6215         }
6216         /*
6217          * In case of ingress traffic when E-Switch mode is enabled,
6218          * we have two cases where we need to set the source port manually.
6219          * The first one, is in case of Nic steering rule, and the second is
6220          * E-Switch rule where no port_id item was found. In both cases
6221          * the source port is set according the current port in use.
6222          */
6223         if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
6224             (priv->representor || priv->master)) {
6225                 if (flow_dv_translate_item_port_id(dev, match_mask,
6226                                                    match_value, NULL))
6227                         return -rte_errno;
6228         }
6229         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
6230                                          dev_flow->dv.value.buf));
6231         dev_flow->layers = item_flags;
6232         /* Register matcher. */
6233         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
6234                                     matcher.mask.size);
6235         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
6236                                                      matcher.priority);
6237         matcher.egress = attr->egress;
6238         matcher.group = dev_flow->group;
6239         matcher.transfer = attr->transfer;
6240         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
6241                 return -rte_errno;
6242         return 0;
6243 }
6244
6245 /**
6246  * Apply the flow to the NIC, lock free,
6247  * (mutex should be acquired by caller).
6248  *
6249  * @param[in] dev
6250  *   Pointer to the Ethernet device structure.
6251  * @param[in, out] flow
6252  *   Pointer to flow structure.
6253  * @param[out] error
6254  *   Pointer to error structure.
6255  *
6256  * @return
6257  *   0 on success, a negative errno value otherwise and rte_errno is set.
6258  */
6259 static int
6260 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
6261                 struct rte_flow_error *error)
6262 {
6263         struct mlx5_flow_dv *dv;
6264         struct mlx5_flow *dev_flow;
6265         struct mlx5_priv *priv = dev->data->dev_private;
6266         int n;
6267         int err;
6268
6269         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
6270                 dv = &dev_flow->dv;
6271                 n = dv->actions_n;
6272                 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) {
6273                         if (dev_flow->transfer) {
6274                                 dv->actions[n++] = priv->sh->esw_drop_action;
6275                         } else {
6276                                 dv->hrxq = mlx5_hrxq_drop_new(dev);
6277                                 if (!dv->hrxq) {
6278                                         rte_flow_error_set
6279                                                 (error, errno,
6280                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6281                                                  NULL,
6282                                                  "cannot get drop hash queue");
6283                                         goto error;
6284                                 }
6285                                 dv->actions[n++] = dv->hrxq->action;
6286                         }
6287                 } else if (dev_flow->actions &
6288                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
6289                         struct mlx5_hrxq *hrxq;
6290
6291                         assert(flow->rss.queue);
6292                         hrxq = mlx5_hrxq_get(dev, flow->rss.key,
6293                                              MLX5_RSS_HASH_KEY_LEN,
6294                                              dev_flow->hash_fields,
6295                                              (*flow->rss.queue),
6296                                              flow->rss.queue_num);
6297                         if (!hrxq) {
6298                                 hrxq = mlx5_hrxq_new
6299                                         (dev, flow->rss.key,
6300                                          MLX5_RSS_HASH_KEY_LEN,
6301                                          dev_flow->hash_fields,
6302                                          (*flow->rss.queue),
6303                                          flow->rss.queue_num,
6304                                          !!(dev_flow->layers &
6305                                             MLX5_FLOW_LAYER_TUNNEL));
6306                         }
6307                         if (!hrxq) {
6308                                 rte_flow_error_set
6309                                         (error, rte_errno,
6310                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6311                                          "cannot get hash queue");
6312                                 goto error;
6313                         }
6314                         dv->hrxq = hrxq;
6315                         dv->actions[n++] = dv->hrxq->action;
6316                 }
6317                 dv->flow =
6318                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
6319                                                   (void *)&dv->value, n,
6320                                                   dv->actions);
6321                 if (!dv->flow) {
6322                         rte_flow_error_set(error, errno,
6323                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6324                                            NULL,
6325                                            "hardware refuses to create flow");
6326                         goto error;
6327                 }
6328                 if (priv->vmwa_context &&
6329                     dev_flow->dv.vf_vlan.tag &&
6330                     !dev_flow->dv.vf_vlan.created) {
6331                         /*
6332                          * The rule contains the VLAN pattern.
6333                          * For VF we are going to create VLAN
6334                          * interface to make hypervisor set correct
6335                          * e-Switch vport context.
6336                          */
6337                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
6338                 }
6339         }
6340         return 0;
6341 error:
6342         err = rte_errno; /* Save rte_errno before cleanup. */
6343         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
6344                 struct mlx5_flow_dv *dv = &dev_flow->dv;
6345                 if (dv->hrxq) {
6346                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
6347                                 mlx5_hrxq_drop_release(dev);
6348                         else
6349                                 mlx5_hrxq_release(dev, dv->hrxq);
6350                         dv->hrxq = NULL;
6351                 }
6352                 if (dev_flow->dv.vf_vlan.tag &&
6353                     dev_flow->dv.vf_vlan.created)
6354                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
6355         }
6356         rte_errno = err; /* Restore rte_errno. */
6357         return -rte_errno;
6358 }
6359
6360 /**
6361  * Release the flow matcher.
6362  *
6363  * @param dev
6364  *   Pointer to Ethernet device.
6365  * @param flow
6366  *   Pointer to mlx5_flow.
6367  *
6368  * @return
6369  *   1 while a reference on it exists, 0 when freed.
6370  */
6371 static int
6372 flow_dv_matcher_release(struct rte_eth_dev *dev,
6373                         struct mlx5_flow *flow)
6374 {
6375         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
6376         struct mlx5_priv *priv = dev->data->dev_private;
6377         struct mlx5_ibv_shared *sh = priv->sh;
6378         struct mlx5_flow_tbl_resource *tbl;
6379
6380         assert(matcher->matcher_object);
6381         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
6382                 dev->data->port_id, (void *)matcher,
6383                 rte_atomic32_read(&matcher->refcnt));
6384         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
6385                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
6386                            (matcher->matcher_object));
6387                 LIST_REMOVE(matcher, next);
6388                 if (matcher->egress)
6389                         tbl = &sh->tx_tbl[matcher->group];
6390                 else
6391                         tbl = &sh->rx_tbl[matcher->group];
6392                 flow_dv_tbl_resource_release(tbl);
6393                 rte_free(matcher);
6394                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
6395                         dev->data->port_id, (void *)matcher);
6396                 return 0;
6397         }
6398         return 1;
6399 }
6400
6401 /**
6402  * Release an encap/decap resource.
6403  *
6404  * @param flow
6405  *   Pointer to mlx5_flow.
6406  *
6407  * @return
6408  *   1 while a reference on it exists, 0 when freed.
6409  */
6410 static int
6411 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
6412 {
6413         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
6414                                                 flow->dv.encap_decap;
6415
6416         assert(cache_resource->verbs_action);
6417         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
6418                 (void *)cache_resource,
6419                 rte_atomic32_read(&cache_resource->refcnt));
6420         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
6421                 claim_zero(mlx5_glue->destroy_flow_action
6422                                 (cache_resource->verbs_action));
6423                 LIST_REMOVE(cache_resource, next);
6424                 rte_free(cache_resource);
6425                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
6426                         (void *)cache_resource);
6427                 return 0;
6428         }
6429         return 1;
6430 }
6431
6432 /**
6433  * Release an jump to table action resource.
6434  *
6435  * @param flow
6436  *   Pointer to mlx5_flow.
6437  *
6438  * @return
6439  *   1 while a reference on it exists, 0 when freed.
6440  */
6441 static int
6442 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
6443 {
6444         struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
6445                                                 flow->dv.jump;
6446
6447         assert(cache_resource->action);
6448         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
6449                 (void *)cache_resource,
6450                 rte_atomic32_read(&cache_resource->refcnt));
6451         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
6452                 claim_zero(mlx5_glue->destroy_flow_action
6453                                 (cache_resource->action));
6454                 LIST_REMOVE(cache_resource, next);
6455                 flow_dv_tbl_resource_release(cache_resource->tbl);
6456                 rte_free(cache_resource);
6457                 DRV_LOG(DEBUG, "jump table resource %p: removed",
6458                         (void *)cache_resource);
6459                 return 0;
6460         }
6461         return 1;
6462 }
6463
6464 /**
6465  * Release a modify-header resource.
6466  *
6467  * @param flow
6468  *   Pointer to mlx5_flow.
6469  *
6470  * @return
6471  *   1 while a reference on it exists, 0 when freed.
6472  */
6473 static int
6474 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
6475 {
6476         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
6477                                                 flow->dv.modify_hdr;
6478
6479         assert(cache_resource->verbs_action);
6480         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
6481                 (void *)cache_resource,
6482                 rte_atomic32_read(&cache_resource->refcnt));
6483         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
6484                 claim_zero(mlx5_glue->destroy_flow_action
6485                                 (cache_resource->verbs_action));
6486                 LIST_REMOVE(cache_resource, next);
6487                 rte_free(cache_resource);
6488                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
6489                         (void *)cache_resource);
6490                 return 0;
6491         }
6492         return 1;
6493 }
6494
6495 /**
6496  * Release port ID action resource.
6497  *
6498  * @param flow
6499  *   Pointer to mlx5_flow.
6500  *
6501  * @return
6502  *   1 while a reference on it exists, 0 when freed.
6503  */
6504 static int
6505 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
6506 {
6507         struct mlx5_flow_dv_port_id_action_resource *cache_resource =
6508                 flow->dv.port_id_action;
6509
6510         assert(cache_resource->action);
6511         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
6512                 (void *)cache_resource,
6513                 rte_atomic32_read(&cache_resource->refcnt));
6514         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
6515                 claim_zero(mlx5_glue->destroy_flow_action
6516                                 (cache_resource->action));
6517                 LIST_REMOVE(cache_resource, next);
6518                 rte_free(cache_resource);
6519                 DRV_LOG(DEBUG, "port id action resource %p: removed",
6520                         (void *)cache_resource);
6521                 return 0;
6522         }
6523         return 1;
6524 }
6525
6526 /**
6527  * Release push vlan action resource.
6528  *
6529  * @param flow
6530  *   Pointer to mlx5_flow.
6531  *
6532  * @return
6533  *   1 while a reference on it exists, 0 when freed.
6534  */
6535 static int
6536 flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow)
6537 {
6538         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource =
6539                 flow->dv.push_vlan_res;
6540
6541         assert(cache_resource->action);
6542         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
6543                 (void *)cache_resource,
6544                 rte_atomic32_read(&cache_resource->refcnt));
6545         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
6546                 claim_zero(mlx5_glue->destroy_flow_action
6547                                 (cache_resource->action));
6548                 LIST_REMOVE(cache_resource, next);
6549                 rte_free(cache_resource);
6550                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
6551                         (void *)cache_resource);
6552                 return 0;
6553         }
6554         return 1;
6555 }
6556
6557 /**
6558  * Remove the flow from the NIC but keeps it in memory.
6559  * Lock free, (mutex should be acquired by caller).
6560  *
6561  * @param[in] dev
6562  *   Pointer to Ethernet device.
6563  * @param[in, out] flow
6564  *   Pointer to flow structure.
6565  */
6566 static void
6567 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
6568 {
6569         struct mlx5_flow_dv *dv;
6570         struct mlx5_flow *dev_flow;
6571
6572         if (!flow)
6573                 return;
6574         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
6575                 dv = &dev_flow->dv;
6576                 if (dv->flow) {
6577                         claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
6578                         dv->flow = NULL;
6579                 }
6580                 if (dv->hrxq) {
6581                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
6582                                 mlx5_hrxq_drop_release(dev);
6583                         else
6584                                 mlx5_hrxq_release(dev, dv->hrxq);
6585                         dv->hrxq = NULL;
6586                 }
6587                 if (dev_flow->dv.vf_vlan.tag &&
6588                     dev_flow->dv.vf_vlan.created)
6589                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
6590         }
6591 }
6592
6593 /**
6594  * Remove the flow from the NIC and the memory.
6595  * Lock free, (mutex should be acquired by caller).
6596  *
6597  * @param[in] dev
6598  *   Pointer to the Ethernet device structure.
6599  * @param[in, out] flow
6600  *   Pointer to flow structure.
6601  */
6602 static void
6603 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
6604 {
6605         struct mlx5_flow *dev_flow;
6606
6607         if (!flow)
6608                 return;
6609         __flow_dv_remove(dev, flow);
6610         if (flow->counter) {
6611                 flow_dv_counter_release(dev, flow->counter);
6612                 flow->counter = NULL;
6613         }
6614         while (!LIST_EMPTY(&flow->dev_flows)) {
6615                 dev_flow = LIST_FIRST(&flow->dev_flows);
6616                 LIST_REMOVE(dev_flow, next);
6617                 if (dev_flow->dv.matcher)
6618                         flow_dv_matcher_release(dev, dev_flow);
6619                 if (dev_flow->dv.encap_decap)
6620                         flow_dv_encap_decap_resource_release(dev_flow);
6621                 if (dev_flow->dv.modify_hdr)
6622                         flow_dv_modify_hdr_resource_release(dev_flow);
6623                 if (dev_flow->dv.jump)
6624                         flow_dv_jump_tbl_resource_release(dev_flow);
6625                 if (dev_flow->dv.port_id_action)
6626                         flow_dv_port_id_action_resource_release(dev_flow);
6627                 if (dev_flow->dv.push_vlan_res)
6628                         flow_dv_push_vlan_action_resource_release(dev_flow);
6629                 if (dev_flow->dv.tag_resource)
6630                         flow_dv_tag_release(dev, dev_flow->dv.tag_resource);
6631                 rte_free(dev_flow);
6632         }
6633 }
6634
6635 /**
6636  * Query a dv flow  rule for its statistics via devx.
6637  *
6638  * @param[in] dev
6639  *   Pointer to Ethernet device.
6640  * @param[in] flow
6641  *   Pointer to the sub flow.
6642  * @param[out] data
6643  *   data retrieved by the query.
6644  * @param[out] error
6645  *   Perform verbose error reporting if not NULL.
6646  *
6647  * @return
6648  *   0 on success, a negative errno value otherwise and rte_errno is set.
6649  */
6650 static int
6651 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
6652                     void *data, struct rte_flow_error *error)
6653 {
6654         struct mlx5_priv *priv = dev->data->dev_private;
6655         struct rte_flow_query_count *qc = data;
6656
6657         if (!priv->config.devx)
6658                 return rte_flow_error_set(error, ENOTSUP,
6659                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6660                                           NULL,
6661                                           "counters are not supported");
6662         if (flow->counter) {
6663                 uint64_t pkts, bytes;
6664                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
6665                                                &bytes);
6666
6667                 if (err)
6668                         return rte_flow_error_set(error, -err,
6669                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6670                                         NULL, "cannot read counters");
6671                 qc->hits_set = 1;
6672                 qc->bytes_set = 1;
6673                 qc->hits = pkts - flow->counter->hits;
6674                 qc->bytes = bytes - flow->counter->bytes;
6675                 if (qc->reset) {
6676                         flow->counter->hits = pkts;
6677                         flow->counter->bytes = bytes;
6678                 }
6679                 return 0;
6680         }
6681         return rte_flow_error_set(error, EINVAL,
6682                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6683                                   NULL,
6684                                   "counters are not available");
6685 }
6686
6687 /**
6688  * Query a flow.
6689  *
6690  * @see rte_flow_query()
6691  * @see rte_flow_ops
6692  */
6693 static int
6694 flow_dv_query(struct rte_eth_dev *dev,
6695               struct rte_flow *flow __rte_unused,
6696               const struct rte_flow_action *actions __rte_unused,
6697               void *data __rte_unused,
6698               struct rte_flow_error *error __rte_unused)
6699 {
6700         int ret = -EINVAL;
6701
6702         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
6703                 switch (actions->type) {
6704                 case RTE_FLOW_ACTION_TYPE_VOID:
6705                         break;
6706                 case RTE_FLOW_ACTION_TYPE_COUNT:
6707                         ret = flow_dv_query_count(dev, flow, data, error);
6708                         break;
6709                 default:
6710                         return rte_flow_error_set(error, ENOTSUP,
6711                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6712                                                   actions,
6713                                                   "action not supported");
6714                 }
6715         }
6716         return ret;
6717 }
6718
6719 /*
6720  * Mutex-protected thunk to lock-free  __flow_dv_translate().
6721  */
6722 static int
6723 flow_dv_translate(struct rte_eth_dev *dev,
6724                   struct mlx5_flow *dev_flow,
6725                   const struct rte_flow_attr *attr,
6726                   const struct rte_flow_item items[],
6727                   const struct rte_flow_action actions[],
6728                   struct rte_flow_error *error)
6729 {
6730         int ret;
6731
6732         flow_dv_shared_lock(dev);
6733         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
6734         flow_dv_shared_unlock(dev);
6735         return ret;
6736 }
6737
6738 /*
6739  * Mutex-protected thunk to lock-free  __flow_dv_apply().
6740  */
6741 static int
6742 flow_dv_apply(struct rte_eth_dev *dev,
6743               struct rte_flow *flow,
6744               struct rte_flow_error *error)
6745 {
6746         int ret;
6747
6748         flow_dv_shared_lock(dev);
6749         ret = __flow_dv_apply(dev, flow, error);
6750         flow_dv_shared_unlock(dev);
6751         return ret;
6752 }
6753
6754 /*
6755  * Mutex-protected thunk to lock-free __flow_dv_remove().
6756  */
6757 static void
6758 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
6759 {
6760         flow_dv_shared_lock(dev);
6761         __flow_dv_remove(dev, flow);
6762         flow_dv_shared_unlock(dev);
6763 }
6764
6765 /*
6766  * Mutex-protected thunk to lock-free __flow_dv_destroy().
6767  */
6768 static void
6769 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
6770 {
6771         flow_dv_shared_lock(dev);
6772         __flow_dv_destroy(dev, flow);
6773         flow_dv_shared_unlock(dev);
6774 }
6775
6776 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
6777         .validate = flow_dv_validate,
6778         .prepare = flow_dv_prepare,
6779         .translate = flow_dv_translate,
6780         .apply = flow_dv_apply,
6781         .remove = flow_dv_remove,
6782         .destroy = flow_dv_destroy,
6783         .query = flow_dv_query,
6784 };
6785
6786 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */