drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_prm.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 union flow_dv_attr {
  54         struct {
  55                 uint32_t valid:1;
  56                 uint32_t ipv4:1;
  57                 uint32_t ipv6:1;
  58                 uint32_t tcp:1;
  59                 uint32_t udp:1;
  60                 uint32_t reserved:27;
  61         };
  62         uint32_t attr;
  63 };
  64
  65 /**
  66  * Initialize flow attributes structure according to flow items' types.
  67  *
  68  * @param[in] item
  69  *   Pointer to item specification.
  70  * @param[out] attr
  71  *   Pointer to flow attributes structure.
  72  */
  73 static void
  74 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  75 {
  76         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  77                 switch (item->type) {
  78                 case RTE_FLOW_ITEM_TYPE_IPV4:
  79                         attr->ipv4 = 1;
  80                         break;
  81                 case RTE_FLOW_ITEM_TYPE_IPV6:
  82                         attr->ipv6 = 1;
  83                         break;
  84                 case RTE_FLOW_ITEM_TYPE_UDP:
  85                         attr->udp = 1;
  86                         break;
  87                 case RTE_FLOW_ITEM_TYPE_TCP:
  88                         attr->tcp = 1;
  89                         break;
  90                 default:
  91                         break;
  92                 }
  93         }
  94         attr->valid = 1;
  95 }
  96
  97 struct field_modify_info {
  98         uint32_t size; /* Size of field in protocol header, in bytes. */
  99         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 100         enum mlx5_modification_field id;
 101 };
 102
 103 struct field_modify_info modify_eth[] = {
 104         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 105         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 106         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 107         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 108         {0, 0, 0},
 109 };
 110
 111 struct field_modify_info modify_ipv4[] = {
 112         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 113         {4, 12, MLX5_MODI_OUT_SIPV4},
 114         {4, 16, MLX5_MODI_OUT_DIPV4},
 115         {0, 0, 0},
 116 };
 117
 118 struct field_modify_info modify_ipv6[] = {
 119         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 120         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 121         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 122         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 123         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 124         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 125         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 126         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 127         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 128         {0, 0, 0},
 129 };
 130
 131 struct field_modify_info modify_udp[] = {
 132         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 133         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 134         {0, 0, 0},
 135 };
 136
 137 struct field_modify_info modify_tcp[] = {
 138         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 139         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 140         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 141         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 142         {0, 0, 0},
 143 };
 144
 145 static void
 146 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 147                           uint8_t next_protocol, uint64_t *item_flags,
 148                           int *tunnel)
 149 {
 150         assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 151                item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 152         if (next_protocol == IPPROTO_IPIP) {
 153                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 154                 *tunnel = 1;
 155         }
 156         if (next_protocol == IPPROTO_IPV6) {
 157                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 158                 *tunnel = 1;
 159         }
 160 }
 161
 162 /**
 163  * Acquire the synchronizing object to protect multithreaded access
 164  * to shared dv context. Lock occurs only if context is actually
 165  * shared, i.e. we have multiport IB device and representors are
 166  * created.
 167  *
 168  * @param[in] dev
 169  *   Pointer to the rte_eth_dev structure.
 170  */
 171 static void
 172 flow_d_shared_lock(struct rte_eth_dev *dev)
 173 {
 174         struct mlx5_priv *priv = dev->data->dev_private;
 175         struct mlx5_ibv_shared *sh = priv->sh;
 176
 177         if (sh->dv_refcnt > 1) {
 178                 int ret;
 179
 180                 ret = pthread_mutex_lock(&sh->dv_mutex);
 181                 assert(!ret);
 182                 (void)ret;
 183         }
 184 }
 185
 186 static void
 187 flow_d_shared_unlock(struct rte_eth_dev *dev)
 188 {
 189         struct mlx5_priv *priv = dev->data->dev_private;
 190         struct mlx5_ibv_shared *sh = priv->sh;
 191
 192         if (sh->dv_refcnt > 1) {
 193                 int ret;
 194
 195                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 196                 assert(!ret);
 197                 (void)ret;
 198         }
 199 }
 200
 201 /**
 202  * Convert modify-header action to DV specification.
 203  *
 204  * @param[in] item
 205  *   Pointer to item specification.
 206  * @param[in] field
 207  *   Pointer to field modification information.
 208  * @param[in,out] resource
 209  *   Pointer to the modify-header resource.
 210  * @param[in] type
 211  *   Type of modification.
 212  * @param[out] error
 213  *   Pointer to the error structure.
 214  *
 215  * @return
 216  *   0 on success, a negative errno value otherwise and rte_errno is set.
 217  */
 218 static int
 219 flow_dv_convert_modify_action(struct rte_flow_item *item,
 220                               struct field_modify_info *field,
 221                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 222                               uint32_t type,
 223                               struct rte_flow_error *error)
 224 {
 225         uint32_t i = resource->actions_num;
 226         struct mlx5_modification_cmd *actions = resource->actions;
 227         const uint8_t *spec = item->spec;
 228         const uint8_t *mask = item->mask;
 229         uint32_t set;
 230
 231         while (field->size) {
 232                 set = 0;
 233                 /* Generate modify command for each mask segment. */
 234                 memcpy(&set, &mask[field->offset], field->size);
 235                 if (set) {
 236                         if (i >= MLX5_MODIFY_NUM)
 237                                 return rte_flow_error_set(error, EINVAL,
 238                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 239                                          "too many items to modify");
 240                         actions[i].action_type = type;
 241                         actions[i].field = field->id;
 242                         actions[i].length = field->size ==
 243                                         4 ? 0 : field->size * 8;
 244                         rte_memcpy(&actions[i].data[4 - field->size],
 245                                    &spec[field->offset], field->size);
 246                         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 247                         ++i;
 248                 }
 249                 if (resource->actions_num != i)
 250                         resource->actions_num = i;
 251                 field++;
 252         }
 253         if (!resource->actions_num)
 254                 return rte_flow_error_set(error, EINVAL,
 255                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 256                                           "invalid modification flow item");
 257         return 0;
 258 }
 259
 260 /**
 261  * Convert modify-header set IPv4 address action to DV specification.
 262  *
 263  * @param[in,out] resource
 264  *   Pointer to the modify-header resource.
 265  * @param[in] action
 266  *   Pointer to action specification.
 267  * @param[out] error
 268  *   Pointer to the error structure.
 269  *
 270  * @return
 271  *   0 on success, a negative errno value otherwise and rte_errno is set.
 272  */
 273 static int
 274 flow_dv_convert_action_modify_ipv4
 275                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 276                          const struct rte_flow_action *action,
 277                          struct rte_flow_error *error)
 278 {
 279         const struct rte_flow_action_set_ipv4 *conf =
 280                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 281         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 282         struct rte_flow_item_ipv4 ipv4;
 283         struct rte_flow_item_ipv4 ipv4_mask;
 284
 285         memset(&ipv4, 0, sizeof(ipv4));
 286         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 287         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 288                 ipv4.hdr.src_addr = conf->ipv4_addr;
 289                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 290         } else {
 291                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 292                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 293         }
 294         item.spec = &ipv4;
 295         item.mask = &ipv4_mask;
 296         return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
 297                                              MLX5_MODIFICATION_TYPE_SET, error);
 298 }
 299
 300 /**
 301  * Convert modify-header set IPv6 address action to DV specification.
 302  *
 303  * @param[in,out] resource
 304  *   Pointer to the modify-header resource.
 305  * @param[in] action
 306  *   Pointer to action specification.
 307  * @param[out] error
 308  *   Pointer to the error structure.
 309  *
 310  * @return
 311  *   0 on success, a negative errno value otherwise and rte_errno is set.
 312  */
 313 static int
 314 flow_dv_convert_action_modify_ipv6
 315                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 316                          const struct rte_flow_action *action,
 317                          struct rte_flow_error *error)
 318 {
 319         const struct rte_flow_action_set_ipv6 *conf =
 320                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 321         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 322         struct rte_flow_item_ipv6 ipv6;
 323         struct rte_flow_item_ipv6 ipv6_mask;
 324
 325         memset(&ipv6, 0, sizeof(ipv6));
 326         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 327         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 328                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 329                        sizeof(ipv6.hdr.src_addr));
 330                 memcpy(&ipv6_mask.hdr.src_addr,
 331                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 332                        sizeof(ipv6.hdr.src_addr));
 333         } else {
 334                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 335                        sizeof(ipv6.hdr.dst_addr));
 336                 memcpy(&ipv6_mask.hdr.dst_addr,
 337                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 338                        sizeof(ipv6.hdr.dst_addr));
 339         }
 340         item.spec = &ipv6;
 341         item.mask = &ipv6_mask;
 342         return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
 343                                              MLX5_MODIFICATION_TYPE_SET, error);
 344 }
 345
 346 /**
 347  * Convert modify-header set MAC address action to DV specification.
 348  *
 349  * @param[in,out] resource
 350  *   Pointer to the modify-header resource.
 351  * @param[in] action
 352  *   Pointer to action specification.
 353  * @param[out] error
 354  *   Pointer to the error structure.
 355  *
 356  * @return
 357  *   0 on success, a negative errno value otherwise and rte_errno is set.
 358  */
 359 static int
 360 flow_dv_convert_action_modify_mac
 361                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 362                          const struct rte_flow_action *action,
 363                          struct rte_flow_error *error)
 364 {
 365         const struct rte_flow_action_set_mac *conf =
 366                 (const struct rte_flow_action_set_mac *)(action->conf);
 367         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 368         struct rte_flow_item_eth eth;
 369         struct rte_flow_item_eth eth_mask;
 370
 371         memset(&eth, 0, sizeof(eth));
 372         memset(&eth_mask, 0, sizeof(eth_mask));
 373         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 374                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 375                        sizeof(eth.src.addr_bytes));
 376                 memcpy(&eth_mask.src.addr_bytes,
 377                        &rte_flow_item_eth_mask.src.addr_bytes,
 378                        sizeof(eth_mask.src.addr_bytes));
 379         } else {
 380                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 381                        sizeof(eth.dst.addr_bytes));
 382                 memcpy(&eth_mask.dst.addr_bytes,
 383                        &rte_flow_item_eth_mask.dst.addr_bytes,
 384                        sizeof(eth_mask.dst.addr_bytes));
 385         }
 386         item.spec = &eth;
 387         item.mask = &eth_mask;
 388         return flow_dv_convert_modify_action(&item, modify_eth, resource,
 389                                              MLX5_MODIFICATION_TYPE_SET, error);
 390 }
 391
 392 /**
 393  * Convert modify-header set TP action to DV specification.
 394  *
 395  * @param[in,out] resource
 396  *   Pointer to the modify-header resource.
 397  * @param[in] action
 398  *   Pointer to action specification.
 399  * @param[in] items
 400  *   Pointer to rte_flow_item objects list.
 401  * @param[in] attr
 402  *   Pointer to flow attributes structure.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   0 on success, a negative errno value otherwise and rte_errno is set.
 408  */
 409 static int
 410 flow_dv_convert_action_modify_tp
 411                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 412                          const struct rte_flow_action *action,
 413                          const struct rte_flow_item *items,
 414                          union flow_dv_attr *attr,
 415                          struct rte_flow_error *error)
 416 {
 417         const struct rte_flow_action_set_tp *conf =
 418                 (const struct rte_flow_action_set_tp *)(action->conf);
 419         struct rte_flow_item item;
 420         struct rte_flow_item_udp udp;
 421         struct rte_flow_item_udp udp_mask;
 422         struct rte_flow_item_tcp tcp;
 423         struct rte_flow_item_tcp tcp_mask;
 424         struct field_modify_info *field;
 425
 426         if (!attr->valid)
 427                 flow_dv_attr_init(items, attr);
 428         if (attr->udp) {
 429                 memset(&udp, 0, sizeof(udp));
 430                 memset(&udp_mask, 0, sizeof(udp_mask));
 431                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 432                         udp.hdr.src_port = conf->port;
 433                         udp_mask.hdr.src_port =
 434                                         rte_flow_item_udp_mask.hdr.src_port;
 435                 } else {
 436                         udp.hdr.dst_port = conf->port;
 437                         udp_mask.hdr.dst_port =
 438                                         rte_flow_item_udp_mask.hdr.dst_port;
 439                 }
 440                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 441                 item.spec = &udp;
 442                 item.mask = &udp_mask;
 443                 field = modify_udp;
 444         }
 445         if (attr->tcp) {
 446                 memset(&tcp, 0, sizeof(tcp));
 447                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 448                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 449                         tcp.hdr.src_port = conf->port;
 450                         tcp_mask.hdr.src_port =
 451                                         rte_flow_item_tcp_mask.hdr.src_port;
 452                 } else {
 453                         tcp.hdr.dst_port = conf->port;
 454                         tcp_mask.hdr.dst_port =
 455                                         rte_flow_item_tcp_mask.hdr.dst_port;
 456                 }
 457                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 458                 item.spec = &tcp;
 459                 item.mask = &tcp_mask;
 460                 field = modify_tcp;
 461         }
 462         return flow_dv_convert_modify_action(&item, field, resource,
 463                                              MLX5_MODIFICATION_TYPE_SET, error);
 464 }
 465
 466 /**
 467  * Convert modify-header set TTL action to DV specification.
 468  *
 469  * @param[in,out] resource
 470  *   Pointer to the modify-header resource.
 471  * @param[in] action
 472  *   Pointer to action specification.
 473  * @param[in] items
 474  *   Pointer to rte_flow_item objects list.
 475  * @param[in] attr
 476  *   Pointer to flow attributes structure.
 477  * @param[out] error
 478  *   Pointer to the error structure.
 479  *
 480  * @return
 481  *   0 on success, a negative errno value otherwise and rte_errno is set.
 482  */
 483 static int
 484 flow_dv_convert_action_modify_ttl
 485                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 486                          const struct rte_flow_action *action,
 487                          const struct rte_flow_item *items,
 488                          union flow_dv_attr *attr,
 489                          struct rte_flow_error *error)
 490 {
 491         const struct rte_flow_action_set_ttl *conf =
 492                 (const struct rte_flow_action_set_ttl *)(action->conf);
 493         struct rte_flow_item item;
 494         struct rte_flow_item_ipv4 ipv4;
 495         struct rte_flow_item_ipv4 ipv4_mask;
 496         struct rte_flow_item_ipv6 ipv6;
 497         struct rte_flow_item_ipv6 ipv6_mask;
 498         struct field_modify_info *field;
 499
 500         if (!attr->valid)
 501                 flow_dv_attr_init(items, attr);
 502         if (attr->ipv4) {
 503                 memset(&ipv4, 0, sizeof(ipv4));
 504                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 505                 ipv4.hdr.time_to_live = conf->ttl_value;
 506                 ipv4_mask.hdr.time_to_live = 0xFF;
 507                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 508                 item.spec = &ipv4;
 509                 item.mask = &ipv4_mask;
 510                 field = modify_ipv4;
 511         }
 512         if (attr->ipv6) {
 513                 memset(&ipv6, 0, sizeof(ipv6));
 514                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 515                 ipv6.hdr.hop_limits = conf->ttl_value;
 516                 ipv6_mask.hdr.hop_limits = 0xFF;
 517                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 518                 item.spec = &ipv6;
 519                 item.mask = &ipv6_mask;
 520                 field = modify_ipv6;
 521         }
 522         return flow_dv_convert_modify_action(&item, field, resource,
 523                                              MLX5_MODIFICATION_TYPE_SET, error);
 524 }
 525
 526 /**
 527  * Convert modify-header decrement TTL action to DV specification.
 528  *
 529  * @param[in,out] resource
 530  *   Pointer to the modify-header resource.
 531  * @param[in] action
 532  *   Pointer to action specification.
 533  * @param[in] items
 534  *   Pointer to rte_flow_item objects list.
 535  * @param[in] attr
 536  *   Pointer to flow attributes structure.
 537  * @param[out] error
 538  *   Pointer to the error structure.
 539  *
 540  * @return
 541  *   0 on success, a negative errno value otherwise and rte_errno is set.
 542  */
 543 static int
 544 flow_dv_convert_action_modify_dec_ttl
 545                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 546                          const struct rte_flow_item *items,
 547                          union flow_dv_attr *attr,
 548                          struct rte_flow_error *error)
 549 {
 550         struct rte_flow_item item;
 551         struct rte_flow_item_ipv4 ipv4;
 552         struct rte_flow_item_ipv4 ipv4_mask;
 553         struct rte_flow_item_ipv6 ipv6;
 554         struct rte_flow_item_ipv6 ipv6_mask;
 555         struct field_modify_info *field;
 556
 557         if (!attr->valid)
 558                 flow_dv_attr_init(items, attr);
 559         if (attr->ipv4) {
 560                 memset(&ipv4, 0, sizeof(ipv4));
 561                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 562                 ipv4.hdr.time_to_live = 0xFF;
 563                 ipv4_mask.hdr.time_to_live = 0xFF;
 564                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 565                 item.spec = &ipv4;
 566                 item.mask = &ipv4_mask;
 567                 field = modify_ipv4;
 568         }
 569         if (attr->ipv6) {
 570                 memset(&ipv6, 0, sizeof(ipv6));
 571                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 572                 ipv6.hdr.hop_limits = 0xFF;
 573                 ipv6_mask.hdr.hop_limits = 0xFF;
 574                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 575                 item.spec = &ipv6;
 576                 item.mask = &ipv6_mask;
 577                 field = modify_ipv6;
 578         }
 579         return flow_dv_convert_modify_action(&item, field, resource,
 580                                              MLX5_MODIFICATION_TYPE_ADD, error);
 581 }
 582
 583 /**
 584  * Convert modify-header increment/decrement TCP Sequence number
 585  * to DV specification.
 586  *
 587  * @param[in,out] resource
 588  *   Pointer to the modify-header resource.
 589  * @param[in] action
 590  *   Pointer to action specification.
 591  * @param[out] error
 592  *   Pointer to the error structure.
 593  *
 594  * @return
 595  *   0 on success, a negative errno value otherwise and rte_errno is set.
 596  */
 597 static int
 598 flow_dv_convert_action_modify_tcp_seq
 599                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 600                          const struct rte_flow_action *action,
 601                          struct rte_flow_error *error)
 602 {
 603         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 604         uint64_t value = rte_be_to_cpu_32(*conf);
 605         struct rte_flow_item item;
 606         struct rte_flow_item_tcp tcp;
 607         struct rte_flow_item_tcp tcp_mask;
 608
 609         memset(&tcp, 0, sizeof(tcp));
 610         memset(&tcp_mask, 0, sizeof(tcp_mask));
 611         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 612                 /*
 613                  * The HW has no decrement operation, only increment operation.
 614                  * To simulate decrement X from Y using increment operation
 615                  * we need to add UINT32_MAX X times to Y.
 616                  * Each adding of UINT32_MAX decrements Y by 1.
 617                  */
 618                 value *= UINT32_MAX;
 619         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 620         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 621         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 622         item.spec = &tcp;
 623         item.mask = &tcp_mask;
 624         return flow_dv_convert_modify_action(&item, modify_tcp, resource,
 625                                              MLX5_MODIFICATION_TYPE_ADD, error);
 626 }
 627
 628 /**
 629  * Convert modify-header increment/decrement TCP Acknowledgment number
 630  * to DV specification.
 631  *
 632  * @param[in,out] resource
 633  *   Pointer to the modify-header resource.
 634  * @param[in] action
 635  *   Pointer to action specification.
 636  * @param[out] error
 637  *   Pointer to the error structure.
 638  *
 639  * @return
 640  *   0 on success, a negative errno value otherwise and rte_errno is set.
 641  */
 642 static int
 643 flow_dv_convert_action_modify_tcp_ack
 644                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 645                          const struct rte_flow_action *action,
 646                          struct rte_flow_error *error)
 647 {
 648         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 649         uint64_t value = rte_be_to_cpu_32(*conf);
 650         struct rte_flow_item item;
 651         struct rte_flow_item_tcp tcp;
 652         struct rte_flow_item_tcp tcp_mask;
 653
 654         memset(&tcp, 0, sizeof(tcp));
 655         memset(&tcp_mask, 0, sizeof(tcp_mask));
 656         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 657                 /*
 658                  * The HW has no decrement operation, only increment operation.
 659                  * To simulate decrement X from Y using increment operation
 660                  * we need to add UINT32_MAX X times to Y.
 661                  * Each adding of UINT32_MAX decrements Y by 1.
 662                  */
 663                 value *= UINT32_MAX;
 664         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 665         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 666         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 667         item.spec = &tcp;
 668         item.mask = &tcp_mask;
 669         return flow_dv_convert_modify_action(&item, modify_tcp, resource,
 670                                              MLX5_MODIFICATION_TYPE_ADD, error);
 671 }
 672
 673 /**
 674  * Validate META item.
 675  *
 676  * @param[in] dev
 677  *   Pointer to the rte_eth_dev structure.
 678  * @param[in] item
 679  *   Item specification.
 680  * @param[in] attr
 681  *   Attributes of flow that includes this item.
 682  * @param[out] error
 683  *   Pointer to error structure.
 684  *
 685  * @return
 686  *   0 on success, a negative errno value otherwise and rte_errno is set.
 687  */
 688 static int
 689 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
 690                            const struct rte_flow_item *item,
 691                            const struct rte_flow_attr *attr,
 692                            struct rte_flow_error *error)
 693 {
 694         const struct rte_flow_item_meta *spec = item->spec;
 695         const struct rte_flow_item_meta *mask = item->mask;
 696         const struct rte_flow_item_meta nic_mask = {
 697                 .data = RTE_BE32(UINT32_MAX)
 698         };
 699         int ret;
 700         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
 701
 702         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
 703                 return rte_flow_error_set(error, EPERM,
 704                                           RTE_FLOW_ERROR_TYPE_ITEM,
 705                                           NULL,
 706                                           "match on metadata offload "
 707                                           "configuration is off for this port");
 708         if (!spec)
 709                 return rte_flow_error_set(error, EINVAL,
 710                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 711                                           item->spec,
 712                                           "data cannot be empty");
 713         if (!spec->data)
 714                 return rte_flow_error_set(error, EINVAL,
 715                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 716                                           NULL,
 717                                           "data cannot be zero");
 718         if (!mask)
 719                 mask = &rte_flow_item_meta_mask;
 720         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 721                                         (const uint8_t *)&nic_mask,
 722                                         sizeof(struct rte_flow_item_meta),
 723                                         error);
 724         if (ret < 0)
 725                 return ret;
 726         if (attr->ingress)
 727                 return rte_flow_error_set(error, ENOTSUP,
 728                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 729                                           NULL,
 730                                           "pattern not supported for ingress");
 731         return 0;
 732 }
 733
 734 /**
 735  * Validate vport item.
 736  *
 737  * @param[in] dev
 738  *   Pointer to the rte_eth_dev structure.
 739  * @param[in] item
 740  *   Item specification.
 741  * @param[in] attr
 742  *   Attributes of flow that includes this item.
 743  * @param[in] item_flags
 744  *   Bit-fields that holds the items detected until now.
 745  * @param[out] error
 746  *   Pointer to error structure.
 747  *
 748  * @return
 749  *   0 on success, a negative errno value otherwise and rte_errno is set.
 750  */
 751 static int
 752 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
 753                               const struct rte_flow_item *item,
 754                               const struct rte_flow_attr *attr,
 755                               uint64_t item_flags,
 756                               struct rte_flow_error *error)
 757 {
 758         const struct rte_flow_item_port_id *spec = item->spec;
 759         const struct rte_flow_item_port_id *mask = item->mask;
 760         const struct rte_flow_item_port_id switch_mask = {
 761                         .id = 0xffffffff,
 762         };
 763         uint16_t esw_domain_id;
 764         uint16_t item_port_esw_domain_id;
 765         int ret;
 766
 767         if (!attr->transfer)
 768                 return rte_flow_error_set(error, EINVAL,
 769                                           RTE_FLOW_ERROR_TYPE_ITEM,
 770                                           NULL,
 771                                           "match on port id is valid only"
 772                                           " when transfer flag is enabled");
 773         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
 774                 return rte_flow_error_set(error, ENOTSUP,
 775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 776                                           "multiple source ports are not"
 777                                           " supported");
 778         if (!mask)
 779                 mask = &switch_mask;
 780         if (mask->id != 0xffffffff)
 781                 return rte_flow_error_set(error, ENOTSUP,
 782                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
 783                                            mask,
 784                                            "no support for partial mask on"
 785                                            " \"id\" field");
 786         ret = mlx5_flow_item_acceptable
 787                                 (item, (const uint8_t *)mask,
 788                                  (const uint8_t *)&rte_flow_item_port_id_mask,
 789                                  sizeof(struct rte_flow_item_port_id),
 790                                  error);
 791         if (ret)
 792                 return ret;
 793         if (!spec)
 794                 return 0;
 795         ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
 796                                         NULL);
 797         if (ret)
 798                 return rte_flow_error_set(error, -ret,
 799                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
 800                                           "failed to obtain E-Switch info for"
 801                                           " port");
 802         ret = mlx5_port_to_eswitch_info(dev->data->port_id,
 803                                         &esw_domain_id, NULL);
 804         if (ret < 0)
 805                 return rte_flow_error_set(error, -ret,
 806                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 807                                           NULL,
 808                                           "failed to obtain E-Switch info");
 809         if (item_port_esw_domain_id != esw_domain_id)
 810                 return rte_flow_error_set(error, -ret,
 811                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
 812                                           "cannot match on a port from a"
 813                                           " different E-Switch");
 814         return 0;
 815 }
 816
 817 /**
 818  * Validate count action.
 819  *
 820  * @param[in] dev
 821  *   device otr.
 822  * @param[out] error
 823  *   Pointer to error structure.
 824  *
 825  * @return
 826  *   0 on success, a negative errno value otherwise and rte_errno is set.
 827  */
 828 static int
 829 flow_dv_validate_action_count(struct rte_eth_dev *dev,
 830                               struct rte_flow_error *error)
 831 {
 832         struct mlx5_priv *priv = dev->data->dev_private;
 833
 834         if (!priv->config.devx)
 835                 goto notsup_err;
 836 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
 837         return 0;
 838 #endif
 839 notsup_err:
 840         return rte_flow_error_set
 841                       (error, ENOTSUP,
 842                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 843                        NULL,
 844                        "count action not supported");
 845 }
 846
 847 /**
 848  * Validate the L2 encap action.
 849  *
 850  * @param[in] action_flags
 851  *   Holds the actions detected until now.
 852  * @param[in] action
 853  *   Pointer to the encap action.
 854  * @param[in] attr
 855  *   Pointer to flow attributes
 856  * @param[out] error
 857  *   Pointer to error structure.
 858  *
 859  * @return
 860  *   0 on success, a negative errno value otherwise and rte_errno is set.
 861  */
 862 static int
 863 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 864                                  const struct rte_flow_action *action,
 865                                  const struct rte_flow_attr *attr,
 866                                  struct rte_flow_error *error)
 867 {
 868         if (!(action->conf))
 869                 return rte_flow_error_set(error, EINVAL,
 870                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 871                                           "configuration cannot be null");
 872         if (action_flags & MLX5_FLOW_ACTION_DROP)
 873                 return rte_flow_error_set(error, EINVAL,
 874                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 875                                           "can't drop and encap in same flow");
 876         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 877                 return rte_flow_error_set(error, EINVAL,
 878                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 879                                           "can only have a single encap or"
 880                                           " decap action in a flow");
 881         if (!attr->transfer && attr->ingress)
 882                 return rte_flow_error_set(error, ENOTSUP,
 883                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 884                                           NULL,
 885                                           "encap action not supported for "
 886                                           "ingress");
 887         return 0;
 888 }
 889
 890 /**
 891  * Validate the L2 decap action.
 892  *
 893  * @param[in] action_flags
 894  *   Holds the actions detected until now.
 895  * @param[in] attr
 896  *   Pointer to flow attributes
 897  * @param[out] error
 898  *   Pointer to error structure.
 899  *
 900  * @return
 901  *   0 on success, a negative errno value otherwise and rte_errno is set.
 902  */
 903 static int
 904 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 905                                  const struct rte_flow_attr *attr,
 906                                  struct rte_flow_error *error)
 907 {
 908         if (action_flags & MLX5_FLOW_ACTION_DROP)
 909                 return rte_flow_error_set(error, EINVAL,
 910                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 911                                           "can't drop and decap in same flow");
 912         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 913                 return rte_flow_error_set(error, EINVAL,
 914                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 915                                           "can only have a single encap or"
 916                                           " decap action in a flow");
 917         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 918                 return rte_flow_error_set(error, EINVAL,
 919                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 920                                           "can't have decap action after"
 921                                           " modify action");
 922         if (attr->egress)
 923                 return rte_flow_error_set(error, ENOTSUP,
 924                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 925                                           NULL,
 926                                           "decap action not supported for "
 927                                           "egress");
 928         return 0;
 929 }
 930
 931 /**
 932  * Validate the raw encap action.
 933  *
 934  * @param[in] action_flags
 935  *   Holds the actions detected until now.
 936  * @param[in] action
 937  *   Pointer to the encap action.
 938  * @param[in] attr
 939  *   Pointer to flow attributes
 940  * @param[out] error
 941  *   Pointer to error structure.
 942  *
 943  * @return
 944  *   0 on success, a negative errno value otherwise and rte_errno is set.
 945  */
 946 static int
 947 flow_dv_validate_action_raw_encap(uint64_t action_flags,
 948                                   const struct rte_flow_action *action,
 949                                   const struct rte_flow_attr *attr,
 950                                   struct rte_flow_error *error)
 951 {
 952         const struct rte_flow_action_raw_encap *raw_encap =
 953                 (const struct rte_flow_action_raw_encap *)action->conf;
 954         if (!(action->conf))
 955                 return rte_flow_error_set(error, EINVAL,
 956                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 957                                           "configuration cannot be null");
 958         if (action_flags & MLX5_FLOW_ACTION_DROP)
 959                 return rte_flow_error_set(error, EINVAL,
 960                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 961                                           "can't drop and encap in same flow");
 962         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 963                 return rte_flow_error_set(error, EINVAL,
 964                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 965                                           "can only have a single encap"
 966                                           " action in a flow");
 967         /* encap without preceding decap is not supported for ingress */
 968         if (!attr->transfer &&  attr->ingress &&
 969             !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
 970                 return rte_flow_error_set(error, ENOTSUP,
 971                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 972                                           NULL,
 973                                           "encap action not supported for "
 974                                           "ingress");
 975         if (!raw_encap->size || !raw_encap->data)
 976                 return rte_flow_error_set(error, EINVAL,
 977                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 978                                           "raw encap data cannot be empty");
 979         return 0;
 980 }
 981
 982 /**
 983  * Validate the raw decap action.
 984  *
 985  * @param[in] action_flags
 986  *   Holds the actions detected until now.
 987  * @param[in] action
 988  *   Pointer to the encap action.
 989  * @param[in] attr
 990  *   Pointer to flow attributes
 991  * @param[out] error
 992  *   Pointer to error structure.
 993  *
 994  * @return
 995  *   0 on success, a negative errno value otherwise and rte_errno is set.
 996  */
 997 static int
 998 flow_dv_validate_action_raw_decap(uint64_t action_flags,
 999                                   const struct rte_flow_action *action,
1000                                   const struct rte_flow_attr *attr,
1001                                   struct rte_flow_error *error)
1002 {
1003         if (action_flags & MLX5_FLOW_ACTION_DROP)
1004                 return rte_flow_error_set(error, EINVAL,
1005                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1006                                           "can't drop and decap in same flow");
1007         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1008                 return rte_flow_error_set(error, EINVAL,
1009                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1010                                           "can't have encap action before"
1011                                           " decap action");
1012         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1013                 return rte_flow_error_set(error, EINVAL,
1014                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1015                                           "can only have a single decap"
1016                                           " action in a flow");
1017         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1018                 return rte_flow_error_set(error, EINVAL,
1019                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1020                                           "can't have decap action after"
1021                                           " modify action");
1022         /* decap action is valid on egress only if it is followed by encap */
1023         if (attr->egress) {
1024                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1025                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1026                        action++) {
1027                 }
1028                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1029                         return rte_flow_error_set
1030                                         (error, ENOTSUP,
1031                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1032                                          NULL, "decap action not supported"
1033                                          " for egress");
1034         }
1035         return 0;
1036 }
1037
1038 /**
1039  * Find existing encap/decap resource or create and register a new one.
1040  *
1041  * @param dev[in, out]
1042  *   Pointer to rte_eth_dev structure.
1043  * @param[in, out] resource
1044  *   Pointer to encap/decap resource.
1045  * @parm[in, out] dev_flow
1046  *   Pointer to the dev_flow.
1047  * @param[out] error
1048  *   pointer to error structure.
1049  *
1050  * @return
1051  *   0 on success otherwise -errno and errno is set.
1052  */
1053 static int
1054 flow_dv_encap_decap_resource_register
1055                         (struct rte_eth_dev *dev,
1056                          struct mlx5_flow_dv_encap_decap_resource *resource,
1057                          struct mlx5_flow *dev_flow,
1058                          struct rte_flow_error *error)
1059 {
1060         struct mlx5_priv *priv = dev->data->dev_private;
1061         struct mlx5_ibv_shared *sh = priv->sh;
1062         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1063         struct rte_flow *flow = dev_flow->flow;
1064         struct mlx5dv_dr_domain *domain;
1065
1066         resource->flags = flow->group ? 0 : 1;
1067         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1068                 domain = sh->fdb_domain;
1069         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1070                 domain = sh->rx_domain;
1071         else
1072                 domain = sh->tx_domain;
1073
1074         /* Lookup a matching resource from cache. */
1075         LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1076                 if (resource->reformat_type == cache_resource->reformat_type &&
1077                     resource->ft_type == cache_resource->ft_type &&
1078                     resource->flags == cache_resource->flags &&
1079                     resource->size == cache_resource->size &&
1080                     !memcmp((const void *)resource->buf,
1081                             (const void *)cache_resource->buf,
1082                             resource->size)) {
1083                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1084                                 (void *)cache_resource,
1085                                 rte_atomic32_read(&cache_resource->refcnt));
1086                         rte_atomic32_inc(&cache_resource->refcnt);
1087                         dev_flow->dv.encap_decap = cache_resource;
1088                         return 0;
1089                 }
1090         }
1091         /* Register new encap/decap resource. */
1092         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1093         if (!cache_resource)
1094                 return rte_flow_error_set(error, ENOMEM,
1095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1096                                           "cannot allocate resource memory");
1097         *cache_resource = *resource;
1098         cache_resource->verbs_action =
1099                 mlx5_glue->dv_create_flow_action_packet_reformat
1100                         (sh->ctx, cache_resource->reformat_type,
1101                          cache_resource->ft_type, domain, cache_resource->flags,
1102                          cache_resource->size,
1103                          (cache_resource->size ? cache_resource->buf : NULL));
1104         if (!cache_resource->verbs_action) {
1105                 rte_free(cache_resource);
1106                 return rte_flow_error_set(error, ENOMEM,
1107                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1108                                           NULL, "cannot create action");
1109         }
1110         rte_atomic32_init(&cache_resource->refcnt);
1111         rte_atomic32_inc(&cache_resource->refcnt);
1112         LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1113         dev_flow->dv.encap_decap = cache_resource;
1114         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1115                 (void *)cache_resource,
1116                 rte_atomic32_read(&cache_resource->refcnt));
1117         return 0;
1118 }
1119
1120 /**
1121  * Find existing table jump resource or create and register a new one.
1122  *
1123  * @param dev[in, out]
1124  *   Pointer to rte_eth_dev structure.
1125  * @param[in, out] resource
1126  *   Pointer to jump table resource.
1127  * @parm[in, out] dev_flow
1128  *   Pointer to the dev_flow.
1129  * @param[out] error
1130  *   pointer to error structure.
1131  *
1132  * @return
1133  *   0 on success otherwise -errno and errno is set.
1134  */
1135 static int
1136 flow_dv_jump_tbl_resource_register
1137                         (struct rte_eth_dev *dev,
1138                          struct mlx5_flow_dv_jump_tbl_resource *resource,
1139                          struct mlx5_flow *dev_flow,
1140                          struct rte_flow_error *error)
1141 {
1142         struct mlx5_priv *priv = dev->data->dev_private;
1143         struct mlx5_ibv_shared *sh = priv->sh;
1144         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1145
1146         /* Lookup a matching resource from cache. */
1147         LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1148                 if (resource->tbl == cache_resource->tbl) {
1149                         DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1150                                 (void *)cache_resource,
1151                                 rte_atomic32_read(&cache_resource->refcnt));
1152                         rte_atomic32_inc(&cache_resource->refcnt);
1153                         dev_flow->dv.jump = cache_resource;
1154                         return 0;
1155                 }
1156         }
1157         /* Register new jump table resource. */
1158         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1159         if (!cache_resource)
1160                 return rte_flow_error_set(error, ENOMEM,
1161                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1162                                           "cannot allocate resource memory");
1163         *cache_resource = *resource;
1164         cache_resource->action =
1165                 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1166                 (resource->tbl->obj);
1167         if (!cache_resource->action) {
1168                 rte_free(cache_resource);
1169                 return rte_flow_error_set(error, ENOMEM,
1170                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1171                                           NULL, "cannot create action");
1172         }
1173         rte_atomic32_init(&cache_resource->refcnt);
1174         rte_atomic32_inc(&cache_resource->refcnt);
1175         LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1176         dev_flow->dv.jump = cache_resource;
1177         DRV_LOG(DEBUG, "new jump table  resource %p: refcnt %d++",
1178                 (void *)cache_resource,
1179                 rte_atomic32_read(&cache_resource->refcnt));
1180         return 0;
1181 }
1182
1183 /**
1184  * Find existing table port ID resource or create and register a new one.
1185  *
1186  * @param dev[in, out]
1187  *   Pointer to rte_eth_dev structure.
1188  * @param[in, out] resource
1189  *   Pointer to port ID action resource.
1190  * @parm[in, out] dev_flow
1191  *   Pointer to the dev_flow.
1192  * @param[out] error
1193  *   pointer to error structure.
1194  *
1195  * @return
1196  *   0 on success otherwise -errno and errno is set.
1197  */
1198 static int
1199 flow_dv_port_id_action_resource_register
1200                         (struct rte_eth_dev *dev,
1201                          struct mlx5_flow_dv_port_id_action_resource *resource,
1202                          struct mlx5_flow *dev_flow,
1203                          struct rte_flow_error *error)
1204 {
1205         struct mlx5_priv *priv = dev->data->dev_private;
1206         struct mlx5_ibv_shared *sh = priv->sh;
1207         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1208
1209         /* Lookup a matching resource from cache. */
1210         LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1211                 if (resource->port_id == cache_resource->port_id) {
1212                         DRV_LOG(DEBUG, "port id action resource resource %p: "
1213                                 "refcnt %d++",
1214                                 (void *)cache_resource,
1215                                 rte_atomic32_read(&cache_resource->refcnt));
1216                         rte_atomic32_inc(&cache_resource->refcnt);
1217                         dev_flow->dv.port_id_action = cache_resource;
1218                         return 0;
1219                 }
1220         }
1221         /* Register new port id action resource. */
1222         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1223         if (!cache_resource)
1224                 return rte_flow_error_set(error, ENOMEM,
1225                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1226                                           "cannot allocate resource memory");
1227         *cache_resource = *resource;
1228         cache_resource->action =
1229                 mlx5_glue->dr_create_flow_action_dest_vport
1230                         (priv->sh->fdb_domain, resource->port_id);
1231         if (!cache_resource->action) {
1232                 rte_free(cache_resource);
1233                 return rte_flow_error_set(error, ENOMEM,
1234                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1235                                           NULL, "cannot create action");
1236         }
1237         rte_atomic32_init(&cache_resource->refcnt);
1238         rte_atomic32_inc(&cache_resource->refcnt);
1239         LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1240         dev_flow->dv.port_id_action = cache_resource;
1241         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1242                 (void *)cache_resource,
1243                 rte_atomic32_read(&cache_resource->refcnt));
1244         return 0;
1245 }
1246
1247 /**
1248  * Get the size of specific rte_flow_item_type
1249  *
1250  * @param[in] item_type
1251  *   Tested rte_flow_item_type.
1252  *
1253  * @return
1254  *   sizeof struct item_type, 0 if void or irrelevant.
1255  */
1256 static size_t
1257 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1258 {
1259         size_t retval;
1260
1261         switch (item_type) {
1262         case RTE_FLOW_ITEM_TYPE_ETH:
1263                 retval = sizeof(struct rte_flow_item_eth);
1264                 break;
1265         case RTE_FLOW_ITEM_TYPE_VLAN:
1266                 retval = sizeof(struct rte_flow_item_vlan);
1267                 break;
1268         case RTE_FLOW_ITEM_TYPE_IPV4:
1269                 retval = sizeof(struct rte_flow_item_ipv4);
1270                 break;
1271         case RTE_FLOW_ITEM_TYPE_IPV6:
1272                 retval = sizeof(struct rte_flow_item_ipv6);
1273                 break;
1274         case RTE_FLOW_ITEM_TYPE_UDP:
1275                 retval = sizeof(struct rte_flow_item_udp);
1276                 break;
1277         case RTE_FLOW_ITEM_TYPE_TCP:
1278                 retval = sizeof(struct rte_flow_item_tcp);
1279                 break;
1280         case RTE_FLOW_ITEM_TYPE_VXLAN:
1281                 retval = sizeof(struct rte_flow_item_vxlan);
1282                 break;
1283         case RTE_FLOW_ITEM_TYPE_GRE:
1284                 retval = sizeof(struct rte_flow_item_gre);
1285                 break;
1286         case RTE_FLOW_ITEM_TYPE_NVGRE:
1287                 retval = sizeof(struct rte_flow_item_nvgre);
1288                 break;
1289         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1290                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1291                 break;
1292         case RTE_FLOW_ITEM_TYPE_MPLS:
1293                 retval = sizeof(struct rte_flow_item_mpls);
1294                 break;
1295         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1296         default:
1297                 retval = 0;
1298                 break;
1299         }
1300         return retval;
1301 }
1302
1303 #define MLX5_ENCAP_IPV4_VERSION         0x40
1304 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
1305 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
1306 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
1307 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
1308 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
1309 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
1310
1311 /**
1312  * Convert the encap action data from list of rte_flow_item to raw buffer
1313  *
1314  * @param[in] items
1315  *   Pointer to rte_flow_item objects list.
1316  * @param[out] buf
1317  *   Pointer to the output buffer.
1318  * @param[out] size
1319  *   Pointer to the output buffer size.
1320  * @param[out] error
1321  *   Pointer to the error structure.
1322  *
1323  * @return
1324  *   0 on success, a negative errno value otherwise and rte_errno is set.
1325  */
1326 static int
1327 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1328                            size_t *size, struct rte_flow_error *error)
1329 {
1330         struct rte_ether_hdr *eth = NULL;
1331         struct rte_vlan_hdr *vlan = NULL;
1332         struct rte_ipv4_hdr *ipv4 = NULL;
1333         struct rte_ipv6_hdr *ipv6 = NULL;
1334         struct rte_udp_hdr *udp = NULL;
1335         struct rte_vxlan_hdr *vxlan = NULL;
1336         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1337         struct rte_gre_hdr *gre = NULL;
1338         size_t len;
1339         size_t temp_size = 0;
1340
1341         if (!items)
1342                 return rte_flow_error_set(error, EINVAL,
1343                                           RTE_FLOW_ERROR_TYPE_ACTION,
1344                                           NULL, "invalid empty data");
1345         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1346                 len = flow_dv_get_item_len(items->type);
1347                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1348                         return rte_flow_error_set(error, EINVAL,
1349                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1350                                                   (void *)items->type,
1351                                                   "items total size is too big"
1352                                                   " for encap action");
1353                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1354                 switch (items->type) {
1355                 case RTE_FLOW_ITEM_TYPE_ETH:
1356                         eth = (struct rte_ether_hdr *)&buf[temp_size];
1357                         break;
1358                 case RTE_FLOW_ITEM_TYPE_VLAN:
1359                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1360                         if (!eth)
1361                                 return rte_flow_error_set(error, EINVAL,
1362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1363                                                 (void *)items->type,
1364                                                 "eth header not found");
1365                         if (!eth->ether_type)
1366                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1367                         break;
1368                 case RTE_FLOW_ITEM_TYPE_IPV4:
1369                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1370                         if (!vlan && !eth)
1371                                 return rte_flow_error_set(error, EINVAL,
1372                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1373                                                 (void *)items->type,
1374                                                 "neither eth nor vlan"
1375                                                 " header found");
1376                         if (vlan && !vlan->eth_proto)
1377                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1378                         else if (eth && !eth->ether_type)
1379                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1380                         if (!ipv4->version_ihl)
1381                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1382                                                     MLX5_ENCAP_IPV4_IHL_MIN;
1383                         if (!ipv4->time_to_live)
1384                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1385                         break;
1386                 case RTE_FLOW_ITEM_TYPE_IPV6:
1387                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1388                         if (!vlan && !eth)
1389                                 return rte_flow_error_set(error, EINVAL,
1390                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1391                                                 (void *)items->type,
1392                                                 "neither eth nor vlan"
1393                                                 " header found");
1394                         if (vlan && !vlan->eth_proto)
1395                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1396                         else if (eth && !eth->ether_type)
1397                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1398                         if (!ipv6->vtc_flow)
1399                                 ipv6->vtc_flow =
1400                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1401                         if (!ipv6->hop_limits)
1402                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1403                         break;
1404                 case RTE_FLOW_ITEM_TYPE_UDP:
1405                         udp = (struct rte_udp_hdr *)&buf[temp_size];
1406                         if (!ipv4 && !ipv6)
1407                                 return rte_flow_error_set(error, EINVAL,
1408                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1409                                                 (void *)items->type,
1410                                                 "ip header not found");
1411                         if (ipv4 && !ipv4->next_proto_id)
1412                                 ipv4->next_proto_id = IPPROTO_UDP;
1413                         else if (ipv6 && !ipv6->proto)
1414                                 ipv6->proto = IPPROTO_UDP;
1415                         break;
1416                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1417                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1418                         if (!udp)
1419                                 return rte_flow_error_set(error, EINVAL,
1420                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1421                                                 (void *)items->type,
1422                                                 "udp header not found");
1423                         if (!udp->dst_port)
1424                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1425                         if (!vxlan->vx_flags)
1426                                 vxlan->vx_flags =
1427                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1428                         break;
1429                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1430                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1431                         if (!udp)
1432                                 return rte_flow_error_set(error, EINVAL,
1433                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1434                                                 (void *)items->type,
1435                                                 "udp header not found");
1436                         if (!vxlan_gpe->proto)
1437                                 return rte_flow_error_set(error, EINVAL,
1438                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1439                                                 (void *)items->type,
1440                                                 "next protocol not found");
1441                         if (!udp->dst_port)
1442                                 udp->dst_port =
1443                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1444                         if (!vxlan_gpe->vx_flags)
1445                                 vxlan_gpe->vx_flags =
1446                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1447                         break;
1448                 case RTE_FLOW_ITEM_TYPE_GRE:
1449                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1450                         gre = (struct rte_gre_hdr *)&buf[temp_size];
1451                         if (!gre->proto)
1452                                 return rte_flow_error_set(error, EINVAL,
1453                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1454                                                 (void *)items->type,
1455                                                 "next protocol not found");
1456                         if (!ipv4 && !ipv6)
1457                                 return rte_flow_error_set(error, EINVAL,
1458                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1459                                                 (void *)items->type,
1460                                                 "ip header not found");
1461                         if (ipv4 && !ipv4->next_proto_id)
1462                                 ipv4->next_proto_id = IPPROTO_GRE;
1463                         else if (ipv6 && !ipv6->proto)
1464                                 ipv6->proto = IPPROTO_GRE;
1465                         break;
1466                 case RTE_FLOW_ITEM_TYPE_VOID:
1467                         break;
1468                 default:
1469                         return rte_flow_error_set(error, EINVAL,
1470                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1471                                                   (void *)items->type,
1472                                                   "unsupported item type");
1473                         break;
1474                 }
1475                 temp_size += len;
1476         }
1477         *size = temp_size;
1478         return 0;
1479 }
1480
1481 static int
1482 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1483 {
1484         struct rte_ether_hdr *eth = NULL;
1485         struct rte_vlan_hdr *vlan = NULL;
1486         struct rte_ipv6_hdr *ipv6 = NULL;
1487         struct rte_udp_hdr *udp = NULL;
1488         char *next_hdr;
1489         uint16_t proto;
1490
1491         eth = (struct rte_ether_hdr *)data;
1492         next_hdr = (char *)(eth + 1);
1493         proto = RTE_BE16(eth->ether_type);
1494
1495         /* VLAN skipping */
1496         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1497                 vlan = (struct rte_vlan_hdr *)next_hdr;
1498                 proto = RTE_BE16(vlan->eth_proto);
1499                 next_hdr += sizeof(struct rte_vlan_hdr);
1500         }
1501
1502         /* HW calculates IPv4 csum. no need to proceed */
1503         if (proto == RTE_ETHER_TYPE_IPV4)
1504                 return 0;
1505
1506         /* non IPv4/IPv6 header. not supported */
1507         if (proto != RTE_ETHER_TYPE_IPV6) {
1508                 return rte_flow_error_set(error, ENOTSUP,
1509                                           RTE_FLOW_ERROR_TYPE_ACTION,
1510                                           NULL, "Cannot offload non IPv4/IPv6");
1511         }
1512
1513         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1514
1515         /* ignore non UDP */
1516         if (ipv6->proto != IPPROTO_UDP)
1517                 return 0;
1518
1519         udp = (struct rte_udp_hdr *)(ipv6 + 1);
1520         udp->dgram_cksum = 0;
1521
1522         return 0;
1523 }
1524
1525 /**
1526  * Convert L2 encap action to DV specification.
1527  *
1528  * @param[in] dev
1529  *   Pointer to rte_eth_dev structure.
1530  * @param[in] action
1531  *   Pointer to action structure.
1532  * @param[in, out] dev_flow
1533  *   Pointer to the mlx5_flow.
1534  * @param[in] transfer
1535  *   Mark if the flow is E-Switch flow.
1536  * @param[out] error
1537  *   Pointer to the error structure.
1538  *
1539  * @return
1540  *   0 on success, a negative errno value otherwise and rte_errno is set.
1541  */
1542 static int
1543 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1544                                const struct rte_flow_action *action,
1545                                struct mlx5_flow *dev_flow,
1546                                uint8_t transfer,
1547                                struct rte_flow_error *error)
1548 {
1549         const struct rte_flow_item *encap_data;
1550         const struct rte_flow_action_raw_encap *raw_encap_data;
1551         struct mlx5_flow_dv_encap_decap_resource res = {
1552                 .reformat_type =
1553                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1554                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1555                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1556         };
1557
1558         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1559                 raw_encap_data =
1560                         (const struct rte_flow_action_raw_encap *)action->conf;
1561                 res.size = raw_encap_data->size;
1562                 memcpy(res.buf, raw_encap_data->data, res.size);
1563                 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1564                         return -rte_errno;
1565         } else {
1566                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1567                         encap_data =
1568                                 ((const struct rte_flow_action_vxlan_encap *)
1569                                                 action->conf)->definition;
1570                 else
1571                         encap_data =
1572                                 ((const struct rte_flow_action_nvgre_encap *)
1573                                                 action->conf)->definition;
1574                 if (flow_dv_convert_encap_data(encap_data, res.buf,
1575                                                &res.size, error))
1576                         return -rte_errno;
1577         }
1578         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1579                 return rte_flow_error_set(error, EINVAL,
1580                                           RTE_FLOW_ERROR_TYPE_ACTION,
1581                                           NULL, "can't create L2 encap action");
1582         return 0;
1583 }
1584
1585 /**
1586  * Convert L2 decap action to DV specification.
1587  *
1588  * @param[in] dev
1589  *   Pointer to rte_eth_dev structure.
1590  * @param[in, out] dev_flow
1591  *   Pointer to the mlx5_flow.
1592  * @param[in] transfer
1593  *   Mark if the flow is E-Switch flow.
1594  * @param[out] error
1595  *   Pointer to the error structure.
1596  *
1597  * @return
1598  *   0 on success, a negative errno value otherwise and rte_errno is set.
1599  */
1600 static int
1601 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1602                                struct mlx5_flow *dev_flow,
1603                                uint8_t transfer,
1604                                struct rte_flow_error *error)
1605 {
1606         struct mlx5_flow_dv_encap_decap_resource res = {
1607                 .size = 0,
1608                 .reformat_type =
1609                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1610                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1611                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1612         };
1613
1614         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1615                 return rte_flow_error_set(error, EINVAL,
1616                                           RTE_FLOW_ERROR_TYPE_ACTION,
1617                                           NULL, "can't create L2 decap action");
1618         return 0;
1619 }
1620
1621 /**
1622  * Convert raw decap/encap (L3 tunnel) action to DV specification.
1623  *
1624  * @param[in] dev
1625  *   Pointer to rte_eth_dev structure.
1626  * @param[in] action
1627  *   Pointer to action structure.
1628  * @param[in, out] dev_flow
1629  *   Pointer to the mlx5_flow.
1630  * @param[in] attr
1631  *   Pointer to the flow attributes.
1632  * @param[out] error
1633  *   Pointer to the error structure.
1634  *
1635  * @return
1636  *   0 on success, a negative errno value otherwise and rte_errno is set.
1637  */
1638 static int
1639 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1640                                 const struct rte_flow_action *action,
1641                                 struct mlx5_flow *dev_flow,
1642                                 const struct rte_flow_attr *attr,
1643                                 struct rte_flow_error *error)
1644 {
1645         const struct rte_flow_action_raw_encap *encap_data;
1646         struct mlx5_flow_dv_encap_decap_resource res;
1647
1648         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1649         res.size = encap_data->size;
1650         memcpy(res.buf, encap_data->data, res.size);
1651         res.reformat_type = attr->egress ?
1652                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1653                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1654         if (attr->transfer)
1655                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1656         else
1657                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1658                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1659         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1660                 return rte_flow_error_set(error, EINVAL,
1661                                           RTE_FLOW_ERROR_TYPE_ACTION,
1662                                           NULL, "can't create encap action");
1663         return 0;
1664 }
1665
1666 /**
1667  * Validate the modify-header actions.
1668  *
1669  * @param[in] action_flags
1670  *   Holds the actions detected until now.
1671  * @param[in] action
1672  *   Pointer to the modify action.
1673  * @param[out] error
1674  *   Pointer to error structure.
1675  *
1676  * @return
1677  *   0 on success, a negative errno value otherwise and rte_errno is set.
1678  */
1679 static int
1680 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1681                                    const struct rte_flow_action *action,
1682                                    struct rte_flow_error *error)
1683 {
1684         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1685                 return rte_flow_error_set(error, EINVAL,
1686                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1687                                           NULL, "action configuration not set");
1688         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1689                 return rte_flow_error_set(error, EINVAL,
1690                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1691                                           "can't have encap action before"
1692                                           " modify action");
1693         return 0;
1694 }
1695
1696 /**
1697  * Validate the modify-header MAC address actions.
1698  *
1699  * @param[in] action_flags
1700  *   Holds the actions detected until now.
1701  * @param[in] action
1702  *   Pointer to the modify action.
1703  * @param[in] item_flags
1704  *   Holds the items detected.
1705  * @param[out] error
1706  *   Pointer to error structure.
1707  *
1708  * @return
1709  *   0 on success, a negative errno value otherwise and rte_errno is set.
1710  */
1711 static int
1712 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1713                                    const struct rte_flow_action *action,
1714                                    const uint64_t item_flags,
1715                                    struct rte_flow_error *error)
1716 {
1717         int ret = 0;
1718
1719         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1720         if (!ret) {
1721                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1722                         return rte_flow_error_set(error, EINVAL,
1723                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1724                                                   NULL,
1725                                                   "no L2 item in pattern");
1726         }
1727         return ret;
1728 }
1729
1730 /**
1731  * Validate the modify-header IPv4 address actions.
1732  *
1733  * @param[in] action_flags
1734  *   Holds the actions detected until now.
1735  * @param[in] action
1736  *   Pointer to the modify action.
1737  * @param[in] item_flags
1738  *   Holds the items detected.
1739  * @param[out] error
1740  *   Pointer to error structure.
1741  *
1742  * @return
1743  *   0 on success, a negative errno value otherwise and rte_errno is set.
1744  */
1745 static int
1746 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
1747                                     const struct rte_flow_action *action,
1748                                     const uint64_t item_flags,
1749                                     struct rte_flow_error *error)
1750 {
1751         int ret = 0;
1752
1753         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1754         if (!ret) {
1755                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
1756                         return rte_flow_error_set(error, EINVAL,
1757                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1758                                                   NULL,
1759                                                   "no ipv4 item in pattern");
1760         }
1761         return ret;
1762 }
1763
1764 /**
1765  * Validate the modify-header IPv6 address actions.
1766  *
1767  * @param[in] action_flags
1768  *   Holds the actions detected until now.
1769  * @param[in] action
1770  *   Pointer to the modify action.
1771  * @param[in] item_flags
1772  *   Holds the items detected.
1773  * @param[out] error
1774  *   Pointer to error structure.
1775  *
1776  * @return
1777  *   0 on success, a negative errno value otherwise and rte_errno is set.
1778  */
1779 static int
1780 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
1781                                     const struct rte_flow_action *action,
1782                                     const uint64_t item_flags,
1783                                     struct rte_flow_error *error)
1784 {
1785         int ret = 0;
1786
1787         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1788         if (!ret) {
1789                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
1790                         return rte_flow_error_set(error, EINVAL,
1791                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1792                                                   NULL,
1793                                                   "no ipv6 item in pattern");
1794         }
1795         return ret;
1796 }
1797
1798 /**
1799  * Validate the modify-header TP actions.
1800  *
1801  * @param[in] action_flags
1802  *   Holds the actions detected until now.
1803  * @param[in] action
1804  *   Pointer to the modify action.
1805  * @param[in] item_flags
1806  *   Holds the items detected.
1807  * @param[out] error
1808  *   Pointer to error structure.
1809  *
1810  * @return
1811  *   0 on success, a negative errno value otherwise and rte_errno is set.
1812  */
1813 static int
1814 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
1815                                   const struct rte_flow_action *action,
1816                                   const uint64_t item_flags,
1817                                   struct rte_flow_error *error)
1818 {
1819         int ret = 0;
1820
1821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1822         if (!ret) {
1823                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
1824                         return rte_flow_error_set(error, EINVAL,
1825                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1826                                                   NULL, "no transport layer "
1827                                                   "in pattern");
1828         }
1829         return ret;
1830 }
1831
1832 /**
1833  * Validate the modify-header actions of increment/decrement
1834  * TCP Sequence-number.
1835  *
1836  * @param[in] action_flags
1837  *   Holds the actions detected until now.
1838  * @param[in] action
1839  *   Pointer to the modify action.
1840  * @param[in] item_flags
1841  *   Holds the items detected.
1842  * @param[out] error
1843  *   Pointer to error structure.
1844  *
1845  * @return
1846  *   0 on success, a negative errno value otherwise and rte_errno is set.
1847  */
1848 static int
1849 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
1850                                        const struct rte_flow_action *action,
1851                                        const uint64_t item_flags,
1852                                        struct rte_flow_error *error)
1853 {
1854         int ret = 0;
1855
1856         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1857         if (!ret) {
1858                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1859                         return rte_flow_error_set(error, EINVAL,
1860                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1861                                                   NULL, "no TCP item in"
1862                                                   " pattern");
1863                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
1864                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
1865                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
1866                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
1867                         return rte_flow_error_set(error, EINVAL,
1868                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1869                                                   NULL,
1870                                                   "cannot decrease and increase"
1871                                                   " TCP sequence number"
1872                                                   " at the same time");
1873         }
1874         return ret;
1875 }
1876
1877 /**
1878  * Validate the modify-header actions of increment/decrement
1879  * TCP Acknowledgment number.
1880  *
1881  * @param[in] action_flags
1882  *   Holds the actions detected until now.
1883  * @param[in] action
1884  *   Pointer to the modify action.
1885  * @param[in] item_flags
1886  *   Holds the items detected.
1887  * @param[out] error
1888  *   Pointer to error structure.
1889  *
1890  * @return
1891  *   0 on success, a negative errno value otherwise and rte_errno is set.
1892  */
1893 static int
1894 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
1895                                        const struct rte_flow_action *action,
1896                                        const uint64_t item_flags,
1897                                        struct rte_flow_error *error)
1898 {
1899         int ret = 0;
1900
1901         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1902         if (!ret) {
1903                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1904                         return rte_flow_error_set(error, EINVAL,
1905                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1906                                                   NULL, "no TCP item in"
1907                                                   " pattern");
1908                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
1909                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
1910                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
1911                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
1912                         return rte_flow_error_set(error, EINVAL,
1913                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1914                                                   NULL,
1915                                                   "cannot decrease and increase"
1916                                                   " TCP acknowledgment number"
1917                                                   " at the same time");
1918         }
1919         return ret;
1920 }
1921
1922 /**
1923  * Validate the modify-header TTL actions.
1924  *
1925  * @param[in] action_flags
1926  *   Holds the actions detected until now.
1927  * @param[in] action
1928  *   Pointer to the modify action.
1929  * @param[in] item_flags
1930  *   Holds the items detected.
1931  * @param[out] error
1932  *   Pointer to error structure.
1933  *
1934  * @return
1935  *   0 on success, a negative errno value otherwise and rte_errno is set.
1936  */
1937 static int
1938 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
1939                                    const struct rte_flow_action *action,
1940                                    const uint64_t item_flags,
1941                                    struct rte_flow_error *error)
1942 {
1943         int ret = 0;
1944
1945         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1946         if (!ret) {
1947                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
1948                         return rte_flow_error_set(error, EINVAL,
1949                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1950                                                   NULL,
1951                                                   "no IP protocol in pattern");
1952         }
1953         return ret;
1954 }
1955
1956 /**
1957  * Validate jump action.
1958  *
1959  * @param[in] action
1960  *   Pointer to the modify action.
1961  * @param[in] group
1962  *   The group of the current flow.
1963  * @param[out] error
1964  *   Pointer to error structure.
1965  *
1966  * @return
1967  *   0 on success, a negative errno value otherwise and rte_errno is set.
1968  */
1969 static int
1970 flow_dv_validate_action_jump(const struct rte_flow_action *action,
1971                              uint32_t group,
1972                              struct rte_flow_error *error)
1973 {
1974         if (action->type != RTE_FLOW_ACTION_TYPE_JUMP && !action->conf)
1975                 return rte_flow_error_set(error, EINVAL,
1976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1977                                           NULL, "action configuration not set");
1978         if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
1979                 return rte_flow_error_set(error, EINVAL,
1980                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1981                                           "target group must be higher then"
1982                                           " the current flow group");
1983         return 0;
1984 }
1985
1986 /*
1987  * Validate the port_id action.
1988  *
1989  * @param[in] dev
1990  *   Pointer to rte_eth_dev structure.
1991  * @param[in] action_flags
1992  *   Bit-fields that holds the actions detected until now.
1993  * @param[in] action
1994  *   Port_id RTE action structure.
1995  * @param[in] attr
1996  *   Attributes of flow that includes this action.
1997  * @param[out] error
1998  *   Pointer to error structure.
1999  *
2000  * @return
2001  *   0 on success, a negative errno value otherwise and rte_errno is set.
2002  */
2003 static int
2004 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2005                                 uint64_t action_flags,
2006                                 const struct rte_flow_action *action,
2007                                 const struct rte_flow_attr *attr,
2008                                 struct rte_flow_error *error)
2009 {
2010         const struct rte_flow_action_port_id *port_id;
2011         uint16_t port;
2012         uint16_t esw_domain_id;
2013         uint16_t act_port_domain_id;
2014         int ret;
2015
2016         if (!attr->transfer)
2017                 return rte_flow_error_set(error, ENOTSUP,
2018                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2019                                           NULL,
2020                                           "port id action is valid in transfer"
2021                                           " mode only");
2022         if (!action || !action->conf)
2023                 return rte_flow_error_set(error, ENOTSUP,
2024                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2025                                           NULL,
2026                                           "port id action parameters must be"
2027                                           " specified");
2028         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2029                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2030                 return rte_flow_error_set(error, EINVAL,
2031                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2032                                           "can have only one fate actions in"
2033                                           " a flow");
2034         ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2035                                         &esw_domain_id, NULL);
2036         if (ret < 0)
2037                 return rte_flow_error_set(error, -ret,
2038                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2039                                           NULL,
2040                                           "failed to obtain E-Switch info");
2041         port_id = action->conf;
2042         port = port_id->original ? dev->data->port_id : port_id->id;
2043         ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2044         if (ret)
2045                 return rte_flow_error_set
2046                                 (error, -ret,
2047                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2048                                  "failed to obtain E-Switch port id for port");
2049         if (act_port_domain_id != esw_domain_id)
2050                 return rte_flow_error_set
2051                                 (error, -ret,
2052                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2053                                  "port does not belong to"
2054                                  " E-Switch being configured");
2055         return 0;
2056 }
2057
2058 /**
2059  * Find existing modify-header resource or create and register a new one.
2060  *
2061  * @param dev[in, out]
2062  *   Pointer to rte_eth_dev structure.
2063  * @param[in, out] resource
2064  *   Pointer to modify-header resource.
2065  * @parm[in, out] dev_flow
2066  *   Pointer to the dev_flow.
2067  * @param[out] error
2068  *   pointer to error structure.
2069  *
2070  * @return
2071  *   0 on success otherwise -errno and errno is set.
2072  */
2073 static int
2074 flow_dv_modify_hdr_resource_register
2075                         (struct rte_eth_dev *dev,
2076                          struct mlx5_flow_dv_modify_hdr_resource *resource,
2077                          struct mlx5_flow *dev_flow,
2078                          struct rte_flow_error *error)
2079 {
2080         struct mlx5_priv *priv = dev->data->dev_private;
2081         struct mlx5_ibv_shared *sh = priv->sh;
2082         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2083         struct mlx5dv_dr_domain *ns;
2084
2085         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2086                 ns = sh->fdb_domain;
2087         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2088                 ns = sh->tx_domain;
2089         else
2090                 ns = sh->rx_domain;
2091         resource->flags =
2092                 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2093         /* Lookup a matching resource from cache. */
2094         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2095                 if (resource->ft_type == cache_resource->ft_type &&
2096                     resource->actions_num == cache_resource->actions_num &&
2097                     resource->flags == cache_resource->flags &&
2098                     !memcmp((const void *)resource->actions,
2099                             (const void *)cache_resource->actions,
2100                             (resource->actions_num *
2101                                             sizeof(resource->actions[0])))) {
2102                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2103                                 (void *)cache_resource,
2104                                 rte_atomic32_read(&cache_resource->refcnt));
2105                         rte_atomic32_inc(&cache_resource->refcnt);
2106                         dev_flow->dv.modify_hdr = cache_resource;
2107                         return 0;
2108                 }
2109         }
2110         /* Register new modify-header resource. */
2111         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2112         if (!cache_resource)
2113                 return rte_flow_error_set(error, ENOMEM,
2114                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2115                                           "cannot allocate resource memory");
2116         *cache_resource = *resource;
2117         cache_resource->verbs_action =
2118                 mlx5_glue->dv_create_flow_action_modify_header
2119                                         (sh->ctx, cache_resource->ft_type,
2120                                          ns, cache_resource->flags,
2121                                          cache_resource->actions_num *
2122                                          sizeof(cache_resource->actions[0]),
2123                                          (uint64_t *)cache_resource->actions);
2124         if (!cache_resource->verbs_action) {
2125                 rte_free(cache_resource);
2126                 return rte_flow_error_set(error, ENOMEM,
2127                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2128                                           NULL, "cannot create action");
2129         }
2130         rte_atomic32_init(&cache_resource->refcnt);
2131         rte_atomic32_inc(&cache_resource->refcnt);
2132         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2133         dev_flow->dv.modify_hdr = cache_resource;
2134         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2135                 (void *)cache_resource,
2136                 rte_atomic32_read(&cache_resource->refcnt));
2137         return 0;
2138 }
2139
2140 #define MLX5_CNT_CONTAINER_RESIZE 64
2141
2142 /**
2143  * Get or create a flow counter.
2144  *
2145  * @param[in] dev
2146  *   Pointer to the Ethernet device structure.
2147  * @param[in] shared
2148  *   Indicate if this counter is shared with other flows.
2149  * @param[in] id
2150  *   Counter identifier.
2151  *
2152  * @return
2153  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2154  */
2155 static struct mlx5_flow_counter *
2156 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2157                                uint32_t id)
2158 {
2159         struct mlx5_priv *priv = dev->data->dev_private;
2160         struct mlx5_flow_counter *cnt = NULL;
2161         struct mlx5_devx_obj *dcs = NULL;
2162
2163         if (!priv->config.devx) {
2164                 rte_errno = ENOTSUP;
2165                 return NULL;
2166         }
2167         if (shared) {
2168                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2169                         if (cnt->shared && cnt->id == id) {
2170                                 cnt->ref_cnt++;
2171                                 return cnt;
2172                         }
2173                 }
2174         }
2175         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2176         if (!dcs)
2177                 return NULL;
2178         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2179         if (!cnt) {
2180                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2181                 rte_errno = ENOMEM;
2182                 return NULL;
2183         }
2184         struct mlx5_flow_counter tmpl = {
2185                 .shared = shared,
2186                 .ref_cnt = 1,
2187                 .id = id,
2188                 .dcs = dcs,
2189         };
2190         tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2191         if (!tmpl.action) {
2192                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2193                 rte_errno = errno;
2194                 rte_free(cnt);
2195                 return NULL;
2196         }
2197         *cnt = tmpl;
2198         TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2199         return cnt;
2200 }
2201
2202 /**
2203  * Release a flow counter.
2204  *
2205  * @param[in] dev
2206  *   Pointer to the Ethernet device structure.
2207  * @param[in] counter
2208  *   Pointer to the counter handler.
2209  */
2210 static void
2211 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2212                                  struct mlx5_flow_counter *counter)
2213 {
2214         struct mlx5_priv *priv = dev->data->dev_private;
2215
2216         if (!counter)
2217                 return;
2218         if (--counter->ref_cnt == 0) {
2219                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2220                 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2221                 rte_free(counter);
2222         }
2223 }
2224
2225 /**
2226  * Query a devx flow counter.
2227  *
2228  * @param[in] dev
2229  *   Pointer to the Ethernet device structure.
2230  * @param[in] cnt
2231  *   Pointer to the flow counter.
2232  * @param[out] pkts
2233  *   The statistics value of packets.
2234  * @param[out] bytes
2235  *   The statistics value of bytes.
2236  *
2237  * @return
2238  *   0 on success, otherwise a negative errno value and rte_errno is set.
2239  */
2240 static inline int
2241 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2242                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2243                      uint64_t *bytes)
2244 {
2245         return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2246                                                 0, NULL, NULL, 0);
2247 }
2248
2249 /**
2250  * Get a pool by a counter.
2251  *
2252  * @param[in] cnt
2253  *   Pointer to the counter.
2254  *
2255  * @return
2256  *   The counter pool.
2257  */
2258 static struct mlx5_flow_counter_pool *
2259 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2260 {
2261         if (!cnt->batch) {
2262                 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2263                 return (struct mlx5_flow_counter_pool *)cnt - 1;
2264         }
2265         return cnt->pool;
2266 }
2267
2268 /**
2269  * Get a pool by devx counter ID.
2270  *
2271  * @param[in] cont
2272  *   Pointer to the counter container.
2273  * @param[in] id
2274  *   The counter devx ID.
2275  *
2276  * @return
2277  *   The counter pool pointer if exists, NULL otherwise,
2278  */
2279 static struct mlx5_flow_counter_pool *
2280 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2281 {
2282         struct mlx5_flow_counter_pool *pool;
2283
2284         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2285                 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2286                                 MLX5_COUNTERS_PER_POOL;
2287
2288                 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2289                         return pool;
2290         };
2291         return NULL;
2292 }
2293
2294 /**
2295  * Allocate a new memory for the counter values wrapped by all the needed
2296  * management.
2297  *
2298  * @param[in] dev
2299  *   Pointer to the Ethernet device structure.
2300  * @param[in] raws_n
2301  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2302  *
2303  * @return
2304  *   The new memory management pointer on success, otherwise NULL and rte_errno
2305  *   is set.
2306  */
2307 static struct mlx5_counter_stats_mem_mng *
2308 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2309 {
2310         struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2311                                         (dev->data->dev_private))->sh;
2312         struct mlx5_devx_mkey_attr mkey_attr;
2313         struct mlx5_counter_stats_mem_mng *mem_mng;
2314         volatile struct flow_counter_stats *raw_data;
2315         int size = (sizeof(struct flow_counter_stats) *
2316                         MLX5_COUNTERS_PER_POOL +
2317                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2318                         sizeof(struct mlx5_counter_stats_mem_mng);
2319         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2320         int i;
2321
2322         if (!mem) {
2323                 rte_errno = ENOMEM;
2324                 return NULL;
2325         }
2326         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2327         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2328         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2329                                                  IBV_ACCESS_LOCAL_WRITE);
2330         if (!mem_mng->umem) {
2331                 rte_errno = errno;
2332                 rte_free(mem);
2333                 return NULL;
2334         }
2335         mkey_attr.addr = (uintptr_t)mem;
2336         mkey_attr.size = size;
2337         mkey_attr.umem_id = mem_mng->umem->umem_id;
2338         mkey_attr.pd = sh->pdn;
2339         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2340         if (!mem_mng->dm) {
2341                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
2342                 rte_errno = errno;
2343                 rte_free(mem);
2344                 return NULL;
2345         }
2346         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
2347         raw_data = (volatile struct flow_counter_stats *)mem;
2348         for (i = 0; i < raws_n; ++i) {
2349                 mem_mng->raws[i].mem_mng = mem_mng;
2350                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
2351         }
2352         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
2353         return mem_mng;
2354 }
2355
2356 /**
2357  * Resize a counter container.
2358  *
2359  * @param[in] dev
2360  *   Pointer to the Ethernet device structure.
2361  * @param[in] batch
2362  *   Whether the pool is for counter that was allocated by batch command.
2363  *
2364  * @return
2365  *   The new container pointer on success, otherwise NULL and rte_errno is set.
2366  */
2367 static struct mlx5_pools_container *
2368 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
2369 {
2370         struct mlx5_priv *priv = dev->data->dev_private;
2371         struct mlx5_pools_container *cont =
2372                         MLX5_CNT_CONTAINER(priv->sh, batch, 0);
2373         struct mlx5_pools_container *new_cont =
2374                         MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
2375         struct mlx5_counter_stats_mem_mng *mem_mng;
2376         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
2377         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
2378         int i;
2379
2380         if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
2381                 /* The last resize still hasn't detected by the host thread. */
2382                 rte_errno = EAGAIN;
2383                 return NULL;
2384         }
2385         new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
2386         if (!new_cont->pools) {
2387                 rte_errno = ENOMEM;
2388                 return NULL;
2389         }
2390         if (cont->n)
2391                 memcpy(new_cont->pools, cont->pools, cont->n *
2392                        sizeof(struct mlx5_flow_counter_pool *));
2393         mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
2394                 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
2395         if (!mem_mng) {
2396                 rte_free(new_cont->pools);
2397                 return NULL;
2398         }
2399         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
2400                 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
2401                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
2402                                  i, next);
2403         new_cont->n = resize;
2404         rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
2405         TAILQ_INIT(&new_cont->pool_list);
2406         TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
2407         new_cont->init_mem_mng = mem_mng;
2408         rte_cio_wmb();
2409          /* Flip the master container. */
2410         priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
2411         return new_cont;
2412 }
2413
2414 /**
2415  * Query a devx flow counter.
2416  *
2417  * @param[in] dev
2418  *   Pointer to the Ethernet device structure.
2419  * @param[in] cnt
2420  *   Pointer to the flow counter.
2421  * @param[out] pkts
2422  *   The statistics value of packets.
2423  * @param[out] bytes
2424  *   The statistics value of bytes.
2425  *
2426  * @return
2427  *   0 on success, otherwise a negative errno value and rte_errno is set.
2428  */
2429 static inline int
2430 _flow_dv_query_count(struct rte_eth_dev *dev,
2431                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2432                      uint64_t *bytes)
2433 {
2434         struct mlx5_priv *priv = dev->data->dev_private;
2435         struct mlx5_flow_counter_pool *pool =
2436                         flow_dv_counter_pool_get(cnt);
2437         int offset = cnt - &pool->counters_raw[0];
2438
2439         if (priv->counter_fallback)
2440                 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
2441
2442         rte_spinlock_lock(&pool->sl);
2443         /*
2444          * The single counters allocation may allocate smaller ID than the
2445          * current allocated in parallel to the host reading.
2446          * In this case the new counter values must be reported as 0.
2447          */
2448         if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
2449                 *pkts = 0;
2450                 *bytes = 0;
2451         } else {
2452                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
2453                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
2454         }
2455         rte_spinlock_unlock(&pool->sl);
2456         return 0;
2457 }
2458
2459 /**
2460  * Create and initialize a new counter pool.
2461  *
2462  * @param[in] dev
2463  *   Pointer to the Ethernet device structure.
2464  * @param[out] dcs
2465  *   The devX counter handle.
2466  * @param[in] batch
2467  *   Whether the pool is for counter that was allocated by batch command.
2468  *
2469  * @return
2470  *   A new pool pointer on success, NULL otherwise and rte_errno is set.
2471  */
2472 static struct mlx5_flow_counter_pool *
2473 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
2474                     uint32_t batch)
2475 {
2476         struct mlx5_priv *priv = dev->data->dev_private;
2477         struct mlx5_flow_counter_pool *pool;
2478         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2479                                                                0);
2480         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
2481         uint32_t size;
2482
2483         if (cont->n == n_valid) {
2484                 cont = flow_dv_container_resize(dev, batch);
2485                 if (!cont)
2486                         return NULL;
2487         }
2488         size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
2489                         sizeof(struct mlx5_flow_counter);
2490         pool = rte_calloc(__func__, 1, size, 0);
2491         if (!pool) {
2492                 rte_errno = ENOMEM;
2493                 return NULL;
2494         }
2495         pool->min_dcs = dcs;
2496         pool->raw = cont->init_mem_mng->raws + n_valid %
2497                                                      MLX5_CNT_CONTAINER_RESIZE;
2498         pool->raw_hw = NULL;
2499         rte_spinlock_init(&pool->sl);
2500         /*
2501          * The generation of the new allocated counters in this pool is 0, 2 in
2502          * the pool generation makes all the counters valid for allocation.
2503          */
2504         rte_atomic64_set(&pool->query_gen, 0x2);
2505         TAILQ_INIT(&pool->counters);
2506         TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2507         cont->pools[n_valid] = pool;
2508         /* Pool initialization must be updated before host thread access. */
2509         rte_cio_wmb();
2510         rte_atomic16_add(&cont->n_valid, 1);
2511         return pool;
2512 }
2513
2514 /**
2515  * Prepare a new counter and/or a new counter pool.
2516  *
2517  * @param[in] dev
2518  *   Pointer to the Ethernet device structure.
2519  * @param[out] cnt_free
2520  *   Where to put the pointer of a new counter.
2521  * @param[in] batch
2522  *   Whether the pool is for counter that was allocated by batch command.
2523  *
2524  * @return
2525  *   The free counter pool pointer and @p cnt_free is set on success,
2526  *   NULL otherwise and rte_errno is set.
2527  */
2528 static struct mlx5_flow_counter_pool *
2529 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
2530                              struct mlx5_flow_counter **cnt_free,
2531                              uint32_t batch)
2532 {
2533         struct mlx5_priv *priv = dev->data->dev_private;
2534         struct mlx5_flow_counter_pool *pool;
2535         struct mlx5_devx_obj *dcs = NULL;
2536         struct mlx5_flow_counter *cnt;
2537         uint32_t i;
2538
2539         if (!batch) {
2540                 /* bulk_bitmap must be 0 for single counter allocation. */
2541                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2542                 if (!dcs)
2543                         return NULL;
2544                 pool = flow_dv_find_pool_by_id
2545                         (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
2546                 if (!pool) {
2547                         pool = flow_dv_pool_create(dev, dcs, batch);
2548                         if (!pool) {
2549                                 mlx5_devx_cmd_destroy(dcs);
2550                                 return NULL;
2551                         }
2552                 } else if (dcs->id < pool->min_dcs->id) {
2553                         rte_atomic64_set(&pool->a64_dcs,
2554                                          (int64_t)(uintptr_t)dcs);
2555                 }
2556                 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
2557                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2558                 cnt->dcs = dcs;
2559                 *cnt_free = cnt;
2560                 return pool;
2561         }
2562         /* bulk_bitmap is in 128 counters units. */
2563         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
2564                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
2565         if (!dcs) {
2566                 rte_errno = ENODATA;
2567                 return NULL;
2568         }
2569         pool = flow_dv_pool_create(dev, dcs, batch);
2570         if (!pool) {
2571                 mlx5_devx_cmd_destroy(dcs);
2572                 return NULL;
2573         }
2574         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2575                 cnt = &pool->counters_raw[i];
2576                 cnt->pool = pool;
2577                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2578         }
2579         *cnt_free = &pool->counters_raw[0];
2580         return pool;
2581 }
2582
2583 /**
2584  * Search for existed shared counter.
2585  *
2586  * @param[in] cont
2587  *   Pointer to the relevant counter pool container.
2588  * @param[in] id
2589  *   The shared counter ID to search.
2590  *
2591  * @return
2592  *   NULL if not existed, otherwise pointer to the shared counter.
2593  */
2594 static struct mlx5_flow_counter *
2595 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
2596                               uint32_t id)
2597 {
2598         static struct mlx5_flow_counter *cnt;
2599         struct mlx5_flow_counter_pool *pool;
2600         int i;
2601
2602         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2603                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2604                         cnt = &pool->counters_raw[i];
2605                         if (cnt->ref_cnt && cnt->shared && cnt->id == id)
2606                                 return cnt;
2607                 }
2608         }
2609         return NULL;
2610 }
2611
2612 /**
2613  * Allocate a flow counter.
2614  *
2615  * @param[in] dev
2616  *   Pointer to the Ethernet device structure.
2617  * @param[in] shared
2618  *   Indicate if this counter is shared with other flows.
2619  * @param[in] id
2620  *   Counter identifier.
2621  * @param[in] group
2622  *   Counter flow group.
2623  *
2624  * @return
2625  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2626  */
2627 static struct mlx5_flow_counter *
2628 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
2629                       uint16_t group)
2630 {
2631         struct mlx5_priv *priv = dev->data->dev_private;
2632         struct mlx5_flow_counter_pool *pool = NULL;
2633         struct mlx5_flow_counter *cnt_free = NULL;
2634         /*
2635          * Currently group 0 flow counter cannot be assigned to a flow if it is
2636          * not the first one in the batch counter allocation, so it is better
2637          * to allocate counters one by one for these flows in a separate
2638          * container.
2639          * A counter can be shared between different groups so need to take
2640          * shared counters from the single container.
2641          */
2642         uint32_t batch = (group && !shared) ? 1 : 0;
2643         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2644                                                                0);
2645
2646         if (priv->counter_fallback)
2647                 return flow_dv_counter_alloc_fallback(dev, shared, id);
2648         if (!priv->config.devx) {
2649                 rte_errno = ENOTSUP;
2650                 return NULL;
2651         }
2652         if (shared) {
2653                 cnt_free = flow_dv_counter_shared_search(cont, id);
2654                 if (cnt_free) {
2655                         if (cnt_free->ref_cnt + 1 == 0) {
2656                                 rte_errno = E2BIG;
2657                                 return NULL;
2658                         }
2659                         cnt_free->ref_cnt++;
2660                         return cnt_free;
2661                 }
2662         }
2663         /* Pools which has a free counters are in the start. */
2664         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2665                 /*
2666                  * The free counter reset values must be updated between the
2667                  * counter release to the counter allocation, so, at least one
2668                  * query must be done in this time. ensure it by saving the
2669                  * query generation in the release time.
2670                  * The free list is sorted according to the generation - so if
2671                  * the first one is not updated, all the others are not
2672                  * updated too.
2673                  */
2674                 cnt_free = TAILQ_FIRST(&pool->counters);
2675                 if (cnt_free && cnt_free->query_gen + 1 <
2676                     rte_atomic64_read(&pool->query_gen))
2677                         break;
2678                 cnt_free = NULL;
2679         }
2680         if (!cnt_free) {
2681                 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
2682                 if (!pool)
2683                         return NULL;
2684         }
2685         cnt_free->batch = batch;
2686         /* Create a DV counter action only in the first time usage. */
2687         if (!cnt_free->action) {
2688                 uint16_t offset;
2689                 struct mlx5_devx_obj *dcs;
2690
2691                 if (batch) {
2692                         offset = cnt_free - &pool->counters_raw[0];
2693                         dcs = pool->min_dcs;
2694                 } else {
2695                         offset = 0;
2696                         dcs = cnt_free->dcs;
2697                 }
2698                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
2699                                         (dcs->obj, offset);
2700                 if (!cnt_free->action) {
2701                         rte_errno = errno;
2702                         return NULL;
2703                 }
2704         }
2705         /* Update the counter reset values. */
2706         if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
2707                                  &cnt_free->bytes))
2708                 return NULL;
2709         cnt_free->shared = shared;
2710         cnt_free->ref_cnt = 1;
2711         cnt_free->id = id;
2712         if (!priv->sh->cmng.query_thread_on)
2713                 /* Start the asynchronous batch query by the host thread. */
2714                 mlx5_set_query_alarm(priv->sh);
2715         TAILQ_REMOVE(&pool->counters, cnt_free, next);
2716         if (TAILQ_EMPTY(&pool->counters)) {
2717                 /* Move the pool to the end of the container pool list. */
2718                 TAILQ_REMOVE(&cont->pool_list, pool, next);
2719                 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2720         }
2721         return cnt_free;
2722 }
2723
2724 /**
2725  * Release a flow counter.
2726  *
2727  * @param[in] dev
2728  *   Pointer to the Ethernet device structure.
2729  * @param[in] counter
2730  *   Pointer to the counter handler.
2731  */
2732 static void
2733 flow_dv_counter_release(struct rte_eth_dev *dev,
2734                         struct mlx5_flow_counter *counter)
2735 {
2736         struct mlx5_priv *priv = dev->data->dev_private;
2737
2738         if (!counter)
2739                 return;
2740         if (priv->counter_fallback) {
2741                 flow_dv_counter_release_fallback(dev, counter);
2742                 return;
2743         }
2744         if (--counter->ref_cnt == 0) {
2745                 struct mlx5_flow_counter_pool *pool =
2746                                 flow_dv_counter_pool_get(counter);
2747
2748                 /* Put the counter in the end - the last updated one. */
2749                 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
2750                 counter->query_gen = rte_atomic64_read(&pool->query_gen);
2751         }
2752 }
2753
2754 /**
2755  * Verify the @p attributes will be correctly understood by the NIC and store
2756  * them in the @p flow if everything is correct.
2757  *
2758  * @param[in] dev
2759  *   Pointer to dev struct.
2760  * @param[in] attributes
2761  *   Pointer to flow attributes
2762  * @param[out] error
2763  *   Pointer to error structure.
2764  *
2765  * @return
2766  *   0 on success, a negative errno value otherwise and rte_errno is set.
2767  */
2768 static int
2769 flow_dv_validate_attributes(struct rte_eth_dev *dev,
2770                             const struct rte_flow_attr *attributes,
2771                             struct rte_flow_error *error)
2772 {
2773         struct mlx5_priv *priv = dev->data->dev_private;
2774         uint32_t priority_max = priv->config.flow_prio - 1;
2775
2776 #ifndef HAVE_MLX5DV_DR
2777         if (attributes->group)
2778                 return rte_flow_error_set(error, ENOTSUP,
2779                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2780                                           NULL,
2781                                           "groups is not supported");
2782 #endif
2783         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
2784             attributes->priority >= priority_max)
2785                 return rte_flow_error_set(error, ENOTSUP,
2786                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2787                                           NULL,
2788                                           "priority out of range");
2789         if (attributes->transfer) {
2790                 if (!priv->config.dv_esw_en)
2791                         return rte_flow_error_set
2792                                 (error, ENOTSUP,
2793                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2794                                  "E-Switch dr is not supported");
2795                 if (!(priv->representor || priv->master))
2796                         return rte_flow_error_set
2797                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2798                                  NULL, "E-Switch configurationd can only be"
2799                                  " done by a master or a representor device");
2800                 if (attributes->egress)
2801                         return rte_flow_error_set
2802                                 (error, ENOTSUP,
2803                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
2804                                  "egress is not supported");
2805                 if (attributes->group >= MLX5_MAX_TABLES_FDB)
2806                         return rte_flow_error_set
2807                                 (error, EINVAL,
2808                                  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2809                                  NULL, "group must be smaller than "
2810                                  RTE_STR(MLX5_MAX_TABLES_FDB));
2811         }
2812         if (!(attributes->egress ^ attributes->ingress))
2813                 return rte_flow_error_set(error, ENOTSUP,
2814                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
2815                                           "must specify exactly one of "
2816                                           "ingress or egress");
2817         return 0;
2818 }
2819
2820 /**
2821  * Internal validation function. For validating both actions and items.
2822  *
2823  * @param[in] dev
2824  *   Pointer to the rte_eth_dev structure.
2825  * @param[in] attr
2826  *   Pointer to the flow attributes.
2827  * @param[in] items
2828  *   Pointer to the list of items.
2829  * @param[in] actions
2830  *   Pointer to the list of actions.
2831  * @param[out] error
2832  *   Pointer to the error structure.
2833  *
2834  * @return
2835  *   0 on success, a negative errno value otherwise and rte_errno is set.
2836  */
2837 static int
2838 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
2839                  const struct rte_flow_item items[],
2840                  const struct rte_flow_action actions[],
2841                  struct rte_flow_error *error)
2842 {
2843         int ret;
2844         uint64_t action_flags = 0;
2845         uint64_t item_flags = 0;
2846         uint64_t last_item = 0;
2847         uint8_t next_protocol = 0xff;
2848         int actions_n = 0;
2849         const struct rte_flow_item *gre_item = NULL;
2850         struct rte_flow_item_tcp nic_tcp_mask = {
2851                 .hdr = {
2852                         .tcp_flags = 0xFF,
2853                         .src_port = RTE_BE16(UINT16_MAX),
2854                         .dst_port = RTE_BE16(UINT16_MAX),
2855                 }
2856         };
2857
2858         if (items == NULL)
2859                 return -1;
2860         ret = flow_dv_validate_attributes(dev, attr, error);
2861         if (ret < 0)
2862                 return ret;
2863         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2864                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2865                 switch (items->type) {
2866                 case RTE_FLOW_ITEM_TYPE_VOID:
2867                         break;
2868                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2869                         ret = flow_dv_validate_item_port_id
2870                                         (dev, items, attr, item_flags, error);
2871                         if (ret < 0)
2872                                 return ret;
2873                         last_item = MLX5_FLOW_ITEM_PORT_ID;
2874                         break;
2875                 case RTE_FLOW_ITEM_TYPE_ETH:
2876                         ret = mlx5_flow_validate_item_eth(items, item_flags,
2877                                                           error);
2878                         if (ret < 0)
2879                                 return ret;
2880                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2881                                              MLX5_FLOW_LAYER_OUTER_L2;
2882                         break;
2883                 case RTE_FLOW_ITEM_TYPE_VLAN:
2884                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2885                                                            dev, error);
2886                         if (ret < 0)
2887                                 return ret;
2888                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2889                                              MLX5_FLOW_LAYER_OUTER_VLAN;
2890                         break;
2891                 case RTE_FLOW_ITEM_TYPE_IPV4:
2892                         mlx5_flow_tunnel_ip_check(items, next_protocol,
2893                                                   &item_flags, &tunnel);
2894                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2895                                                            NULL, error);
2896                         if (ret < 0)
2897                                 return ret;
2898                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2899                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2900                         if (items->mask != NULL &&
2901                             ((const struct rte_flow_item_ipv4 *)
2902                              items->mask)->hdr.next_proto_id) {
2903                                 next_protocol =
2904                                         ((const struct rte_flow_item_ipv4 *)
2905                                          (items->spec))->hdr.next_proto_id;
2906                                 next_protocol &=
2907                                         ((const struct rte_flow_item_ipv4 *)
2908                                          (items->mask))->hdr.next_proto_id;
2909                         } else {
2910                                 /* Reset for inner layer. */
2911                                 next_protocol = 0xff;
2912                         }
2913                         break;
2914                 case RTE_FLOW_ITEM_TYPE_IPV6:
2915                         mlx5_flow_tunnel_ip_check(items, next_protocol,
2916                                                   &item_flags, &tunnel);
2917                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2918                                                            NULL, error);
2919                         if (ret < 0)
2920                                 return ret;
2921                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2922                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2923                         if (items->mask != NULL &&
2924                             ((const struct rte_flow_item_ipv6 *)
2925                              items->mask)->hdr.proto) {
2926                                 next_protocol =
2927                                         ((const struct rte_flow_item_ipv6 *)
2928                                          items->spec)->hdr.proto;
2929                                 next_protocol &=
2930                                         ((const struct rte_flow_item_ipv6 *)
2931                                          items->mask)->hdr.proto;
2932                         } else {
2933                                 /* Reset for inner layer. */
2934                                 next_protocol = 0xff;
2935                         }
2936                         break;
2937                 case RTE_FLOW_ITEM_TYPE_TCP:
2938                         ret = mlx5_flow_validate_item_tcp
2939                                                 (items, item_flags,
2940                                                  next_protocol,
2941                                                  &nic_tcp_mask,
2942                                                  error);
2943                         if (ret < 0)
2944                                 return ret;
2945                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
2946                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
2947                         break;
2948                 case RTE_FLOW_ITEM_TYPE_UDP:
2949                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2950                                                           next_protocol,
2951                                                           error);
2952                         if (ret < 0)
2953                                 return ret;
2954                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
2955                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
2956                         break;
2957                 case RTE_FLOW_ITEM_TYPE_GRE:
2958                         ret = mlx5_flow_validate_item_gre(items, item_flags,
2959                                                           next_protocol, error);
2960                         if (ret < 0)
2961                                 return ret;
2962                         gre_item = items;
2963                         last_item = MLX5_FLOW_LAYER_GRE;
2964                         break;
2965                 case RTE_FLOW_ITEM_TYPE_NVGRE:
2966                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
2967                                                             next_protocol,
2968                                                             error);
2969                         if (ret < 0)
2970                                 return ret;
2971                         last_item = MLX5_FLOW_LAYER_NVGRE;
2972                         break;
2973                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
2974                         ret = mlx5_flow_validate_item_gre_key
2975                                 (items, item_flags, gre_item, error);
2976                         if (ret < 0)
2977                                 return ret;
2978                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
2979                         break;
2980                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2981                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
2982                                                             error);
2983                         if (ret < 0)
2984                                 return ret;
2985                         last_item = MLX5_FLOW_LAYER_VXLAN;
2986                         break;
2987                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2988                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
2989                                                                 item_flags, dev,
2990                                                                 error);
2991                         if (ret < 0)
2992                                 return ret;
2993                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
2994                         break;
2995                 case RTE_FLOW_ITEM_TYPE_MPLS:
2996                         ret = mlx5_flow_validate_item_mpls(dev, items,
2997                                                            item_flags,
2998                                                            last_item, error);
2999                         if (ret < 0)
3000                                 return ret;
3001                         last_item = MLX5_FLOW_LAYER_MPLS;
3002                         break;
3003                 case RTE_FLOW_ITEM_TYPE_META:
3004                         ret = flow_dv_validate_item_meta(dev, items, attr,
3005                                                          error);
3006                         if (ret < 0)
3007                                 return ret;
3008                         last_item = MLX5_FLOW_ITEM_METADATA;
3009                         break;
3010                 case RTE_FLOW_ITEM_TYPE_ICMP:
3011                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
3012                                                            next_protocol,
3013                                                            error);
3014                         if (ret < 0)
3015                                 return ret;
3016                         last_item = MLX5_FLOW_LAYER_ICMP;
3017                         break;
3018                 case RTE_FLOW_ITEM_TYPE_ICMP6:
3019                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3020                                                             next_protocol,
3021                                                             error);
3022                         if (ret < 0)
3023                                 return ret;
3024                         last_item = MLX5_FLOW_LAYER_ICMP6;
3025                         break;
3026                 default:
3027                         return rte_flow_error_set(error, ENOTSUP,
3028                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3029                                                   NULL, "item not supported");
3030                 }
3031                 item_flags |= last_item;
3032         }
3033         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3034                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3035                         return rte_flow_error_set(error, ENOTSUP,
3036                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3037                                                   actions, "too many actions");
3038                 switch (actions->type) {
3039                 case RTE_FLOW_ACTION_TYPE_VOID:
3040                         break;
3041                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3042                         ret = flow_dv_validate_action_port_id(dev,
3043                                                               action_flags,
3044                                                               actions,
3045                                                               attr,
3046                                                               error);
3047                         if (ret)
3048                                 return ret;
3049                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3050                         ++actions_n;
3051                         break;
3052                 case RTE_FLOW_ACTION_TYPE_FLAG:
3053                         ret = mlx5_flow_validate_action_flag(action_flags,
3054                                                              attr, error);
3055                         if (ret < 0)
3056                                 return ret;
3057                         action_flags |= MLX5_FLOW_ACTION_FLAG;
3058                         ++actions_n;
3059                         break;
3060                 case RTE_FLOW_ACTION_TYPE_MARK:
3061                         ret = mlx5_flow_validate_action_mark(actions,
3062                                                              action_flags,
3063                                                              attr, error);
3064                         if (ret < 0)
3065                                 return ret;
3066                         action_flags |= MLX5_FLOW_ACTION_MARK;
3067                         ++actions_n;
3068                         break;
3069                 case RTE_FLOW_ACTION_TYPE_DROP:
3070                         ret = mlx5_flow_validate_action_drop(action_flags,
3071                                                              attr, error);
3072                         if (ret < 0)
3073                                 return ret;
3074                         action_flags |= MLX5_FLOW_ACTION_DROP;
3075                         ++actions_n;
3076                         break;
3077                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3078                         ret = mlx5_flow_validate_action_queue(actions,
3079                                                               action_flags, dev,
3080                                                               attr, error);
3081                         if (ret < 0)
3082                                 return ret;
3083                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
3084                         ++actions_n;
3085                         break;
3086                 case RTE_FLOW_ACTION_TYPE_RSS:
3087                         ret = mlx5_flow_validate_action_rss(actions,
3088                                                             action_flags, dev,
3089                                                             attr, item_flags,
3090                                                             error);
3091                         if (ret < 0)
3092                                 return ret;
3093                         action_flags |= MLX5_FLOW_ACTION_RSS;
3094                         ++actions_n;
3095                         break;
3096                 case RTE_FLOW_ACTION_TYPE_COUNT:
3097                         ret = flow_dv_validate_action_count(dev, error);
3098                         if (ret < 0)
3099                                 return ret;
3100                         action_flags |= MLX5_FLOW_ACTION_COUNT;
3101                         ++actions_n;
3102                         break;
3103                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3104                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3105                         ret = flow_dv_validate_action_l2_encap(action_flags,
3106                                                                actions, attr,
3107                                                                error);
3108                         if (ret < 0)
3109                                 return ret;
3110                         action_flags |= actions->type ==
3111                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3112                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
3113                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
3114                         ++actions_n;
3115                         break;
3116                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3117                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3118                         ret = flow_dv_validate_action_l2_decap(action_flags,
3119                                                                attr, error);
3120                         if (ret < 0)
3121                                 return ret;
3122                         action_flags |= actions->type ==
3123                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3124                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
3125                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
3126                         ++actions_n;
3127                         break;
3128                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3129                         ret = flow_dv_validate_action_raw_encap(action_flags,
3130                                                                 actions, attr,
3131                                                                 error);
3132                         if (ret < 0)
3133                                 return ret;
3134                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3135                         ++actions_n;
3136                         break;
3137                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3138                         ret = flow_dv_validate_action_raw_decap(action_flags,
3139                                                                 actions, attr,
3140                                                                 error);
3141                         if (ret < 0)
3142                                 return ret;
3143                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3144                         ++actions_n;
3145                         break;
3146                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3147                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3148                         ret = flow_dv_validate_action_modify_mac(action_flags,
3149                                                                  actions,
3150                                                                  item_flags,
3151                                                                  error);
3152                         if (ret < 0)
3153                                 return ret;
3154                         /* Count all modify-header actions as one action. */
3155                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3156                                 ++actions_n;
3157                         action_flags |= actions->type ==
3158                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3159                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
3160                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
3161                         break;
3162
3163                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3164                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3165                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
3166                                                                   actions,
3167                                                                   item_flags,
3168                                                                   error);
3169                         if (ret < 0)
3170                                 return ret;
3171                         /* Count all modify-header actions as one action. */
3172                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3173                                 ++actions_n;
3174                         action_flags |= actions->type ==
3175                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3176                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3177                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
3178                         break;
3179                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3180                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3181                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
3182                                                                   actions,
3183                                                                   item_flags,
3184                                                                   error);
3185                         if (ret < 0)
3186                                 return ret;
3187                         /* Count all modify-header actions as one action. */
3188                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3189                                 ++actions_n;
3190                         action_flags |= actions->type ==
3191                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3192                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3193                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
3194                         break;
3195                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3196                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3197                         ret = flow_dv_validate_action_modify_tp(action_flags,
3198                                                                 actions,
3199                                                                 item_flags,
3200                                                                 error);
3201                         if (ret < 0)
3202                                 return ret;
3203                         /* Count all modify-header actions as one action. */
3204                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3205                                 ++actions_n;
3206                         action_flags |= actions->type ==
3207                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3208                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
3209                                                 MLX5_FLOW_ACTION_SET_TP_DST;
3210                         break;
3211                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3212                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3213                         ret = flow_dv_validate_action_modify_ttl(action_flags,
3214                                                                  actions,
3215                                                                  item_flags,
3216                                                                  error);
3217                         if (ret < 0)
3218                                 return ret;
3219                         /* Count all modify-header actions as one action. */
3220                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3221                                 ++actions_n;
3222                         action_flags |= actions->type ==
3223                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
3224                                                 MLX5_FLOW_ACTION_SET_TTL :
3225                                                 MLX5_FLOW_ACTION_DEC_TTL;
3226                         break;
3227                 case RTE_FLOW_ACTION_TYPE_JUMP:
3228                         ret = flow_dv_validate_action_jump(actions,
3229                                                            attr->group, error);
3230                         if (ret)
3231                                 return ret;
3232                         ++actions_n;
3233                         action_flags |= MLX5_FLOW_ACTION_JUMP;
3234                         break;
3235                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3236                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3237                         ret = flow_dv_validate_action_modify_tcp_seq
3238                                                                 (action_flags,
3239                                                                  actions,
3240                                                                  item_flags,
3241                                                                  error);
3242                         if (ret < 0)
3243                                 return ret;
3244                         /* Count all modify-header actions as one action. */
3245                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3246                                 ++actions_n;
3247                         action_flags |= actions->type ==
3248                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
3249                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
3250                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
3251                         break;
3252                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3253                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3254                         ret = flow_dv_validate_action_modify_tcp_ack
3255                                                                 (action_flags,
3256                                                                  actions,
3257                                                                  item_flags,
3258                                                                  error);
3259                         if (ret < 0)
3260                                 return ret;
3261                         /* Count all modify-header actions as one action. */
3262                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3263                                 ++actions_n;
3264                         action_flags |= actions->type ==
3265                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
3266                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
3267                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
3268                         break;
3269                 default:
3270                         return rte_flow_error_set(error, ENOTSUP,
3271                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3272                                                   actions,
3273                                                   "action not supported");
3274                 }
3275         }
3276         /* Eswitch has few restrictions on using items and actions */
3277         if (attr->transfer) {
3278                 if (action_flags & MLX5_FLOW_ACTION_FLAG)
3279                         return rte_flow_error_set(error, ENOTSUP,
3280                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3281                                                   NULL,
3282                                                   "unsupported action FLAG");
3283                 if (action_flags & MLX5_FLOW_ACTION_MARK)
3284                         return rte_flow_error_set(error, ENOTSUP,
3285                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3286                                                   NULL,
3287                                                   "unsupported action MARK");
3288                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
3289                         return rte_flow_error_set(error, ENOTSUP,
3290                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3291                                                   NULL,
3292                                                   "unsupported action QUEUE");
3293                 if (action_flags & MLX5_FLOW_ACTION_RSS)
3294                         return rte_flow_error_set(error, ENOTSUP,
3295                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3296                                                   NULL,
3297                                                   "unsupported action RSS");
3298                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3299                         return rte_flow_error_set(error, EINVAL,
3300                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3301                                                   actions,
3302                                                   "no fate action is found");
3303         } else {
3304                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
3305                         return rte_flow_error_set(error, EINVAL,
3306                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3307                                                   actions,
3308                                                   "no fate action is found");
3309         }
3310         return 0;
3311 }
3312
3313 /**
3314  * Internal preparation function. Allocates the DV flow size,
3315  * this size is constant.
3316  *
3317  * @param[in] attr
3318  *   Pointer to the flow attributes.
3319  * @param[in] items
3320  *   Pointer to the list of items.
3321  * @param[in] actions
3322  *   Pointer to the list of actions.
3323  * @param[out] error
3324  *   Pointer to the error structure.
3325  *
3326  * @return
3327  *   Pointer to mlx5_flow object on success,
3328  *   otherwise NULL and rte_errno is set.
3329  */
3330 static struct mlx5_flow *
3331 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
3332                 const struct rte_flow_item items[] __rte_unused,
3333                 const struct rte_flow_action actions[] __rte_unused,
3334                 struct rte_flow_error *error)
3335 {
3336         uint32_t size = sizeof(struct mlx5_flow);
3337         struct mlx5_flow *flow;
3338
3339         flow = rte_calloc(__func__, 1, size, 0);
3340         if (!flow) {
3341                 rte_flow_error_set(error, ENOMEM,
3342                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3343                                    "not enough memory to create flow");
3344                 return NULL;
3345         }
3346         flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
3347         return flow;
3348 }
3349
3350 #ifndef NDEBUG
3351 /**
3352  * Sanity check for match mask and value. Similar to check_valid_spec() in
3353  * kernel driver. If unmasked bit is present in value, it returns failure.
3354  *
3355  * @param match_mask
3356  *   pointer to match mask buffer.
3357  * @param match_value
3358  *   pointer to match value buffer.
3359  *
3360  * @return
3361  *   0 if valid, -EINVAL otherwise.
3362  */
3363 static int
3364 flow_dv_check_valid_spec(void *match_mask, void *match_value)
3365 {
3366         uint8_t *m = match_mask;
3367         uint8_t *v = match_value;
3368         unsigned int i;
3369
3370         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
3371                 if (v[i] & ~m[i]) {
3372                         DRV_LOG(ERR,
3373                                 "match_value differs from match_criteria"
3374                                 " %p[%u] != %p[%u]",
3375                                 match_value, i, match_mask, i);
3376                         return -EINVAL;
3377                 }
3378         }
3379         return 0;
3380 }
3381 #endif
3382
3383 /**
3384  * Add Ethernet item to matcher and to the value.
3385  *
3386  * @param[in, out] matcher
3387  *   Flow matcher.
3388  * @param[in, out] key
3389  *   Flow matcher value.
3390  * @param[in] item
3391  *   Flow pattern to translate.
3392  * @param[in] inner
3393  *   Item is inner pattern.
3394  */
3395 static void
3396 flow_dv_translate_item_eth(void *matcher, void *key,
3397                            const struct rte_flow_item *item, int inner)
3398 {
3399         const struct rte_flow_item_eth *eth_m = item->mask;
3400         const struct rte_flow_item_eth *eth_v = item->spec;
3401         const struct rte_flow_item_eth nic_mask = {
3402                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3403                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3404                 .type = RTE_BE16(0xffff),
3405         };
3406         void *headers_m;
3407         void *headers_v;
3408         char *l24_v;
3409         unsigned int i;
3410
3411         if (!eth_v)
3412                 return;
3413         if (!eth_m)
3414                 eth_m = &nic_mask;
3415         if (inner) {
3416                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3417                                          inner_headers);
3418                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3419         } else {
3420                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3421                                          outer_headers);
3422                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3423         }
3424         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
3425                &eth_m->dst, sizeof(eth_m->dst));
3426         /* The value must be in the range of the mask. */
3427         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
3428         for (i = 0; i < sizeof(eth_m->dst); ++i)
3429                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
3430         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
3431                &eth_m->src, sizeof(eth_m->src));
3432         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
3433         /* The value must be in the range of the mask. */
3434         for (i = 0; i < sizeof(eth_m->dst); ++i)
3435                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
3436         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3437                  rte_be_to_cpu_16(eth_m->type));
3438         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
3439         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
3440 }
3441
3442 /**
3443  * Add VLAN item to matcher and to the value.
3444  *
3445  * @param[in, out] dev_flow
3446  *   Flow descriptor.
3447  * @param[in, out] matcher
3448  *   Flow matcher.
3449  * @param[in, out] key
3450  *   Flow matcher value.
3451  * @param[in] item
3452  *   Flow pattern to translate.
3453  * @param[in] inner
3454  *   Item is inner pattern.
3455  */
3456 static void
3457 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
3458                             void *matcher, void *key,
3459                             const struct rte_flow_item *item,
3460                             int inner)
3461 {
3462         const struct rte_flow_item_vlan *vlan_m = item->mask;
3463         const struct rte_flow_item_vlan *vlan_v = item->spec;
3464         void *headers_m;
3465         void *headers_v;
3466         uint16_t tci_m;
3467         uint16_t tci_v;
3468
3469         if (!vlan_v)
3470                 return;
3471         if (!vlan_m)
3472                 vlan_m = &rte_flow_item_vlan_mask;
3473         if (inner) {
3474                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3475                                          inner_headers);
3476                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3477         } else {
3478                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3479                                          outer_headers);
3480                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3481                 /*
3482                  * This is workaround, masks are not supported,
3483                  * and pre-validated.
3484                  */
3485                 dev_flow->dv.vf_vlan.tag =
3486                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
3487         }
3488         tci_m = rte_be_to_cpu_16(vlan_m->tci);
3489         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
3490         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
3491         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
3492         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
3493         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
3494         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
3495         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
3496         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
3497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
3498         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3499                  rte_be_to_cpu_16(vlan_m->inner_type));
3500         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
3501                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
3502 }
3503
3504 /**
3505  * Add IPV4 item to matcher and to the value.
3506  *
3507  * @param[in, out] matcher
3508  *   Flow matcher.
3509  * @param[in, out] key
3510  *   Flow matcher value.
3511  * @param[in] item
3512  *   Flow pattern to translate.
3513  * @param[in] inner
3514  *   Item is inner pattern.
3515  * @param[in] group
3516  *   The group to insert the rule.
3517  */
3518 static void
3519 flow_dv_translate_item_ipv4(void *matcher, void *key,
3520                             const struct rte_flow_item *item,
3521                             int inner, uint32_t group)
3522 {
3523         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
3524         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
3525         const struct rte_flow_item_ipv4 nic_mask = {
3526                 .hdr = {
3527                         .src_addr = RTE_BE32(0xffffffff),
3528                         .dst_addr = RTE_BE32(0xffffffff),
3529                         .type_of_service = 0xff,
3530                         .next_proto_id = 0xff,
3531                 },
3532         };
3533         void *headers_m;
3534         void *headers_v;
3535         char *l24_m;
3536         char *l24_v;
3537         uint8_t tos;
3538
3539         if (inner) {
3540                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3541                                          inner_headers);
3542                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3543         } else {
3544                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3545                                          outer_headers);
3546                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3547         }
3548         if (group == 0)
3549                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3550         else
3551                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
3552         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
3553         if (!ipv4_v)
3554                 return;
3555         if (!ipv4_m)
3556                 ipv4_m = &nic_mask;
3557         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3558                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3559         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3560                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3561         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
3562         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
3563         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3564                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
3565         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3566                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
3567         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
3568         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
3569         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
3570         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
3571                  ipv4_m->hdr.type_of_service);
3572         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
3573         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
3574                  ipv4_m->hdr.type_of_service >> 2);
3575         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
3576         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3577                  ipv4_m->hdr.next_proto_id);
3578         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3579                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3580 }
3581
3582 /**
3583  * Add IPV6 item to matcher and to the value.
3584  *
3585  * @param[in, out] matcher
3586  *   Flow matcher.
3587  * @param[in, out] key
3588  *   Flow matcher value.
3589  * @param[in] item
3590  *   Flow pattern to translate.
3591  * @param[in] inner
3592  *   Item is inner pattern.
3593  * @param[in] group
3594  *   The group to insert the rule.
3595  */
3596 static void
3597 flow_dv_translate_item_ipv6(void *matcher, void *key,
3598                             const struct rte_flow_item *item,
3599                             int inner, uint32_t group)
3600 {
3601         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
3602         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
3603         const struct rte_flow_item_ipv6 nic_mask = {
3604                 .hdr = {
3605                         .src_addr =
3606                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
3607                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
3608                         .dst_addr =
3609                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
3610                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
3611                         .vtc_flow = RTE_BE32(0xffffffff),
3612                         .proto = 0xff,
3613                         .hop_limits = 0xff,
3614                 },
3615         };
3616         void *headers_m;
3617         void *headers_v;
3618         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3619         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3620         char *l24_m;
3621         char *l24_v;
3622         uint32_t vtc_m;
3623         uint32_t vtc_v;
3624         int i;
3625         int size;
3626
3627         if (inner) {
3628                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3629                                          inner_headers);
3630                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3631         } else {
3632                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3633                                          outer_headers);
3634                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3635         }
3636         if (group == 0)
3637                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3638         else
3639                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
3640         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
3641         if (!ipv6_v)
3642                 return;
3643         if (!ipv6_m)
3644                 ipv6_m = &nic_mask;
3645         size = sizeof(ipv6_m->hdr.dst_addr);
3646         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3647                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3648         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3649                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3650         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
3651         for (i = 0; i < size; ++i)
3652                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
3653         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3654                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
3655         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3656                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
3657         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
3658         for (i = 0; i < size; ++i)
3659                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
3660         /* TOS. */
3661         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
3662         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
3663         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
3664         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
3665         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
3666         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
3667         /* Label. */
3668         if (inner) {
3669                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
3670                          vtc_m);
3671                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
3672                          vtc_v);
3673         } else {
3674                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
3675                          vtc_m);
3676                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
3677                          vtc_v);
3678         }
3679         /* Protocol. */
3680         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3681                  ipv6_m->hdr.proto);
3682         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3683                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
3684 }
3685
3686 /**
3687  * Add TCP item to matcher and to the value.
3688  *
3689  * @param[in, out] matcher
3690  *   Flow matcher.
3691  * @param[in, out] key
3692  *   Flow matcher value.
3693  * @param[in] item
3694  *   Flow pattern to translate.
3695  * @param[in] inner
3696  *   Item is inner pattern.
3697  */
3698 static void
3699 flow_dv_translate_item_tcp(void *matcher, void *key,
3700                            const struct rte_flow_item *item,
3701                            int inner)
3702 {
3703         const struct rte_flow_item_tcp *tcp_m = item->mask;
3704         const struct rte_flow_item_tcp *tcp_v = item->spec;
3705         void *headers_m;
3706         void *headers_v;
3707
3708         if (inner) {
3709                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3710                                          inner_headers);
3711                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3712         } else {
3713                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3714                                          outer_headers);
3715                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3716         }
3717         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3718         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
3719         if (!tcp_v)
3720                 return;
3721         if (!tcp_m)
3722                 tcp_m = &rte_flow_item_tcp_mask;
3723         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
3724                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
3725         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
3726                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
3727         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
3728                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
3729         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
3730                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
3731         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
3732                  tcp_m->hdr.tcp_flags);
3733         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
3734                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
3735 }
3736
3737 /**
3738  * Add UDP item to matcher and to the value.
3739  *
3740  * @param[in, out] matcher
3741  *   Flow matcher.
3742  * @param[in, out] key
3743  *   Flow matcher value.
3744  * @param[in] item
3745  *   Flow pattern to translate.
3746  * @param[in] inner
3747  *   Item is inner pattern.
3748  */
3749 static void
3750 flow_dv_translate_item_udp(void *matcher, void *key,
3751                            const struct rte_flow_item *item,
3752                            int inner)
3753 {
3754         const struct rte_flow_item_udp *udp_m = item->mask;
3755         const struct rte_flow_item_udp *udp_v = item->spec;
3756         void *headers_m;
3757         void *headers_v;
3758
3759         if (inner) {
3760                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3761                                          inner_headers);
3762                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3763         } else {
3764                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3765                                          outer_headers);
3766                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3767         }
3768         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3769         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
3770         if (!udp_v)
3771                 return;
3772         if (!udp_m)
3773                 udp_m = &rte_flow_item_udp_mask;
3774         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
3775                  rte_be_to_cpu_16(udp_m->hdr.src_port));
3776         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
3777                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
3778         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
3779                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
3780         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
3781                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
3782 }
3783
3784 /**
3785  * Add GRE optional Key item to matcher and to the value.
3786  *
3787  * @param[in, out] matcher
3788  *   Flow matcher.
3789  * @param[in, out] key
3790  *   Flow matcher value.
3791  * @param[in] item
3792  *   Flow pattern to translate.
3793  * @param[in] inner
3794  *   Item is inner pattern.
3795  */
3796 static void
3797 flow_dv_translate_item_gre_key(void *matcher, void *key,
3798                                    const struct rte_flow_item *item)
3799 {
3800         const rte_be32_t *key_m = item->mask;
3801         const rte_be32_t *key_v = item->spec;
3802         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3803         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3804         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
3805
3806         if (!key_v)
3807                 return;
3808         if (!key_m)
3809                 key_m = &gre_key_default_mask;
3810         /* GRE K bit must be on and should already be validated */
3811         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
3812         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
3813         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
3814                  rte_be_to_cpu_32(*key_m) >> 8);
3815         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
3816                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
3817         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
3818                  rte_be_to_cpu_32(*key_m) & 0xFF);
3819         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
3820                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
3821 }
3822
3823 /**
3824  * Add GRE item to matcher and to the value.
3825  *
3826  * @param[in, out] matcher
3827  *   Flow matcher.
3828  * @param[in, out] key
3829  *   Flow matcher value.
3830  * @param[in] item
3831  *   Flow pattern to translate.
3832  * @param[in] inner
3833  *   Item is inner pattern.
3834  */
3835 static void
3836 flow_dv_translate_item_gre(void *matcher, void *key,
3837                            const struct rte_flow_item *item,
3838                            int inner)
3839 {
3840         const struct rte_flow_item_gre *gre_m = item->mask;
3841         const struct rte_flow_item_gre *gre_v = item->spec;
3842         void *headers_m;
3843         void *headers_v;
3844         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3845         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3846         struct {
3847                 union {
3848                         __extension__
3849                         struct {
3850                                 uint16_t version:3;
3851                                 uint16_t rsvd0:9;
3852                                 uint16_t s_present:1;
3853                                 uint16_t k_present:1;
3854                                 uint16_t rsvd_bit1:1;
3855                                 uint16_t c_present:1;
3856                         };
3857                         uint16_t value;
3858                 };
3859         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
3860
3861         if (inner) {
3862                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3863                                          inner_headers);
3864                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3865         } else {
3866                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3867                                          outer_headers);
3868                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3869         }
3870         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3871         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
3872         if (!gre_v)
3873                 return;
3874         if (!gre_m)
3875                 gre_m = &rte_flow_item_gre_mask;
3876         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
3877                  rte_be_to_cpu_16(gre_m->protocol));
3878         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
3879                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
3880         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
3881         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
3882         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
3883                  gre_crks_rsvd0_ver_m.c_present);
3884         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
3885                  gre_crks_rsvd0_ver_v.c_present &
3886                  gre_crks_rsvd0_ver_m.c_present);
3887         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
3888                  gre_crks_rsvd0_ver_m.k_present);
3889         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
3890                  gre_crks_rsvd0_ver_v.k_present &
3891                  gre_crks_rsvd0_ver_m.k_present);
3892         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
3893                  gre_crks_rsvd0_ver_m.s_present);
3894         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
3895                  gre_crks_rsvd0_ver_v.s_present &
3896                  gre_crks_rsvd0_ver_m.s_present);
3897 }
3898
3899 /**
3900  * Add NVGRE item to matcher and to the value.
3901  *
3902  * @param[in, out] matcher
3903  *   Flow matcher.
3904  * @param[in, out] key
3905  *   Flow matcher value.
3906  * @param[in] item
3907  *   Flow pattern to translate.
3908  * @param[in] inner
3909  *   Item is inner pattern.
3910  */
3911 static void
3912 flow_dv_translate_item_nvgre(void *matcher, void *key,
3913                              const struct rte_flow_item *item,
3914                              int inner)
3915 {
3916         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
3917         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
3918         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3919         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3920         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
3921         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
3922         char *gre_key_m;
3923         char *gre_key_v;
3924         int size;
3925         int i;
3926
3927         /* For NVGRE, GRE header fields must be set with defined values. */
3928         const struct rte_flow_item_gre gre_spec = {
3929                 .c_rsvd0_ver = RTE_BE16(0x2000),
3930                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
3931         };
3932         const struct rte_flow_item_gre gre_mask = {
3933                 .c_rsvd0_ver = RTE_BE16(0xB000),
3934                 .protocol = RTE_BE16(UINT16_MAX),
3935         };
3936         const struct rte_flow_item gre_item = {
3937                 .spec = &gre_spec,
3938                 .mask = &gre_mask,
3939                 .last = NULL,
3940         };
3941         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
3942         if (!nvgre_v)
3943                 return;
3944         if (!nvgre_m)
3945                 nvgre_m = &rte_flow_item_nvgre_mask;
3946         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
3947         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
3948         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
3949         memcpy(gre_key_m, tni_flow_id_m, size);
3950         for (i = 0; i < size; ++i)
3951                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
3952 }
3953
3954 /**
3955  * Add VXLAN item to matcher and to the value.
3956  *
3957  * @param[in, out] matcher
3958  *   Flow matcher.
3959  * @param[in, out] key
3960  *   Flow matcher value.
3961  * @param[in] item
3962  *   Flow pattern to translate.
3963  * @param[in] inner
3964  *   Item is inner pattern.
3965  */
3966 static void
3967 flow_dv_translate_item_vxlan(void *matcher, void *key,
3968                              const struct rte_flow_item *item,
3969                              int inner)
3970 {
3971         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
3972         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
3973         void *headers_m;
3974         void *headers_v;
3975         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3976         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3977         char *vni_m;
3978         char *vni_v;
3979         uint16_t dport;
3980         int size;
3981         int i;
3982
3983         if (inner) {
3984                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3985                                          inner_headers);
3986                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3987         } else {
3988                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3989                                          outer_headers);
3990                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3991         }
3992         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
3993                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
3994         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
3995                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
3996                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
3997         }
3998         if (!vxlan_v)
3999                 return;
4000         if (!vxlan_m)
4001                 vxlan_m = &rte_flow_item_vxlan_mask;
4002         size = sizeof(vxlan_m->vni);
4003         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4004         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4005         memcpy(vni_m, vxlan_m->vni, size);
4006         for (i = 0; i < size; ++i)
4007                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4008 }
4009
4010 /**
4011  * Add MPLS item to matcher and to the value.
4012  *
4013  * @param[in, out] matcher
4014  *   Flow matcher.
4015  * @param[in, out] key
4016  *   Flow matcher value.
4017  * @param[in] item
4018  *   Flow pattern to translate.
4019  * @param[in] prev_layer
4020  *   The protocol layer indicated in previous item.
4021  * @param[in] inner
4022  *   Item is inner pattern.
4023  */
4024 static void
4025 flow_dv_translate_item_mpls(void *matcher, void *key,
4026                             const struct rte_flow_item *item,
4027                             uint64_t prev_layer,
4028                             int inner)
4029 {
4030         const uint32_t *in_mpls_m = item->mask;
4031         const uint32_t *in_mpls_v = item->spec;
4032         uint32_t *out_mpls_m = 0;
4033         uint32_t *out_mpls_v = 0;
4034         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4035         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4036         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4037                                      misc_parameters_2);
4038         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4039         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4040         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4041
4042         switch (prev_layer) {
4043         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4044                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4045                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4046                          MLX5_UDP_PORT_MPLS);
4047                 break;
4048         case MLX5_FLOW_LAYER_GRE:
4049                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4050                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4051                          RTE_ETHER_TYPE_MPLS);
4052                 break;
4053         default:
4054                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4055                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4056                          IPPROTO_MPLS);
4057                 break;
4058         }
4059         if (!in_mpls_v)
4060                 return;
4061         if (!in_mpls_m)
4062                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4063         switch (prev_layer) {
4064         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4065                 out_mpls_m =
4066                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4067                                                  outer_first_mpls_over_udp);
4068                 out_mpls_v =
4069                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4070                                                  outer_first_mpls_over_udp);
4071                 break;
4072         case MLX5_FLOW_LAYER_GRE:
4073                 out_mpls_m =
4074                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4075                                                  outer_first_mpls_over_gre);
4076                 out_mpls_v =
4077                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4078                                                  outer_first_mpls_over_gre);
4079                 break;
4080         default:
4081                 /* Inner MPLS not over GRE is not supported. */
4082                 if (!inner) {
4083                         out_mpls_m =
4084                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4085                                                          misc2_m,
4086                                                          outer_first_mpls);
4087                         out_mpls_v =
4088                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4089                                                          misc2_v,
4090                                                          outer_first_mpls);
4091                 }
4092                 break;
4093         }
4094         if (out_mpls_m && out_mpls_v) {
4095                 *out_mpls_m = *in_mpls_m;
4096                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4097         }
4098 }
4099
4100 /**
4101  * Add META item to matcher
4102  *
4103  * @param[in, out] matcher
4104  *   Flow matcher.
4105  * @param[in, out] key
4106  *   Flow matcher value.
4107  * @param[in] item
4108  *   Flow pattern to translate.
4109  * @param[in] inner
4110  *   Item is inner pattern.
4111  */
4112 static void
4113 flow_dv_translate_item_meta(void *matcher, void *key,
4114                             const struct rte_flow_item *item)
4115 {
4116         const struct rte_flow_item_meta *meta_m;
4117         const struct rte_flow_item_meta *meta_v;
4118         void *misc2_m =
4119                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4120         void *misc2_v =
4121                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4122
4123         meta_m = (const void *)item->mask;
4124         if (!meta_m)
4125                 meta_m = &rte_flow_item_meta_mask;
4126         meta_v = (const void *)item->spec;
4127         if (meta_v) {
4128                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
4129                          rte_be_to_cpu_32(meta_m->data));
4130                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
4131                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
4132         }
4133 }
4134
4135 /**
4136  * Add source vport match to the specified matcher.
4137  *
4138  * @param[in, out] matcher
4139  *   Flow matcher.
4140  * @param[in, out] key
4141  *   Flow matcher value.
4142  * @param[in] port
4143  *   Source vport value to match
4144  * @param[in] mask
4145  *   Mask
4146  */
4147 static void
4148 flow_dv_translate_item_source_vport(void *matcher, void *key,
4149                                     int16_t port, uint16_t mask)
4150 {
4151         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4152         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4153
4154         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
4155         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
4156 }
4157
4158 /**
4159  * Translate port-id item to eswitch match on  port-id.
4160  *
4161  * @param[in] dev
4162  *   The devich to configure through.
4163  * @param[in, out] matcher
4164  *   Flow matcher.
4165  * @param[in, out] key
4166  *   Flow matcher value.
4167  * @param[in] item
4168  *   Flow pattern to translate.
4169  *
4170  * @return
4171  *   0 on success, a negative errno value otherwise.
4172  */
4173 static int
4174 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
4175                                void *key, const struct rte_flow_item *item)
4176 {
4177         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
4178         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
4179         uint16_t mask, val, id;
4180         int ret;
4181
4182         mask = pid_m ? pid_m->id : 0xffff;
4183         id = pid_v ? pid_v->id : dev->data->port_id;
4184         ret = mlx5_port_to_eswitch_info(id, NULL, &val);
4185         if (ret)
4186                 return ret;
4187         flow_dv_translate_item_source_vport(matcher, key, val, mask);
4188         return 0;
4189 }
4190
4191 /**
4192  * Add ICMP6 item to matcher and to the value.
4193  *
4194  * @param[in, out] matcher
4195  *   Flow matcher.
4196  * @param[in, out] key
4197  *   Flow matcher value.
4198  * @param[in] item
4199  *   Flow pattern to translate.
4200  * @param[in] inner
4201  *   Item is inner pattern.
4202  */
4203 static void
4204 flow_dv_translate_item_icmp6(void *matcher, void *key,
4205                               const struct rte_flow_item *item,
4206                               int inner)
4207 {
4208         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
4209         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
4210         void *headers_m;
4211         void *headers_v;
4212         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4213                                      misc_parameters_3);
4214         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4215         if (inner) {
4216                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4217                                          inner_headers);
4218                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4219         } else {
4220                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4221                                          outer_headers);
4222                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4223         }
4224         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4225         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
4226         if (!icmp6_v)
4227                 return;
4228         if (!icmp6_m)
4229                 icmp6_m = &rte_flow_item_icmp6_mask;
4230         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
4231         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
4232                  icmp6_v->type & icmp6_m->type);
4233         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
4234         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
4235                  icmp6_v->code & icmp6_m->code);
4236 }
4237
4238 /**
4239  * Add ICMP item to matcher and to the value.
4240  *
4241  * @param[in, out] matcher
4242  *   Flow matcher.
4243  * @param[in, out] key
4244  *   Flow matcher value.
4245  * @param[in] item
4246  *   Flow pattern to translate.
4247  * @param[in] inner
4248  *   Item is inner pattern.
4249  */
4250 static void
4251 flow_dv_translate_item_icmp(void *matcher, void *key,
4252                             const struct rte_flow_item *item,
4253                             int inner)
4254 {
4255         const struct rte_flow_item_icmp *icmp_m = item->mask;
4256         const struct rte_flow_item_icmp *icmp_v = item->spec;
4257         void *headers_m;
4258         void *headers_v;
4259         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4260                                      misc_parameters_3);
4261         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4262         if (inner) {
4263                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4264                                          inner_headers);
4265                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4266         } else {
4267                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4268                                          outer_headers);
4269                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4270         }
4271         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4272         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
4273         if (!icmp_v)
4274                 return;
4275         if (!icmp_m)
4276                 icmp_m = &rte_flow_item_icmp_mask;
4277         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
4278                  icmp_m->hdr.icmp_type);
4279         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
4280                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
4281         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
4282                  icmp_m->hdr.icmp_code);
4283         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
4284                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
4285 }
4286
4287 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
4288
4289 #define HEADER_IS_ZERO(match_criteria, headers)                              \
4290         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
4291                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
4292
4293 /**
4294  * Calculate flow matcher enable bitmap.
4295  *
4296  * @param match_criteria
4297  *   Pointer to flow matcher criteria.
4298  *
4299  * @return
4300  *   Bitmap of enabled fields.
4301  */
4302 static uint8_t
4303 flow_dv_matcher_enable(uint32_t *match_criteria)
4304 {
4305         uint8_t match_criteria_enable;
4306
4307         match_criteria_enable =
4308                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
4309                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
4310         match_criteria_enable |=
4311                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
4312                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
4313         match_criteria_enable |=
4314                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
4315                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
4316         match_criteria_enable |=
4317                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
4318                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
4319         match_criteria_enable |=
4320                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
4321                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
4322         return match_criteria_enable;
4323 }
4324
4325
4326 /**
4327  * Get a flow table.
4328  *
4329  * @param dev[in, out]
4330  *   Pointer to rte_eth_dev structure.
4331  * @param[in] table_id
4332  *   Table id to use.
4333  * @param[in] egress
4334  *   Direction of the table.
4335  * @param[in] transfer
4336  *   E-Switch or NIC flow.
4337  * @param[out] error
4338  *   pointer to error structure.
4339  *
4340  * @return
4341  *   Returns tables resource based on the index, NULL in case of failed.
4342  */
4343 static struct mlx5_flow_tbl_resource *
4344 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
4345                          uint32_t table_id, uint8_t egress,
4346                          uint8_t transfer,
4347                          struct rte_flow_error *error)
4348 {
4349         struct mlx5_priv *priv = dev->data->dev_private;
4350         struct mlx5_ibv_shared *sh = priv->sh;
4351         struct mlx5_flow_tbl_resource *tbl;
4352
4353 #ifdef HAVE_MLX5DV_DR
4354         if (transfer) {
4355                 tbl = &sh->fdb_tbl[table_id];
4356                 if (!tbl->obj)
4357                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4358                                 (sh->fdb_domain, table_id);
4359         } else if (egress) {
4360                 tbl = &sh->tx_tbl[table_id];
4361                 if (!tbl->obj)
4362                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4363                                 (sh->tx_domain, table_id);
4364         } else {
4365                 tbl = &sh->rx_tbl[table_id];
4366                 if (!tbl->obj)
4367                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4368                                 (sh->rx_domain, table_id);
4369         }
4370         if (!tbl->obj) {
4371                 rte_flow_error_set(error, ENOMEM,
4372                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4373                                    NULL, "cannot create table");
4374                 return NULL;
4375         }
4376         rte_atomic32_inc(&tbl->refcnt);
4377         return tbl;
4378 #else
4379         (void)error;
4380         (void)tbl;
4381         if (transfer)
4382                 return &sh->fdb_tbl[table_id];
4383         else if (egress)
4384                 return &sh->tx_tbl[table_id];
4385         else
4386                 return &sh->rx_tbl[table_id];
4387 #endif
4388 }
4389
4390 /**
4391  * Release a flow table.
4392  *
4393  * @param[in] tbl
4394  *   Table resource to be released.
4395  *
4396  * @return
4397  *   Returns 0 if table was released, else return 1;
4398  */
4399 static int
4400 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
4401 {
4402         if (!tbl)
4403                 return 0;
4404         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
4405                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
4406                 tbl->obj = NULL;
4407                 return 0;
4408         }
4409         return 1;
4410 }
4411
4412 /**
4413  * Register the flow matcher.
4414  *
4415  * @param dev[in, out]
4416  *   Pointer to rte_eth_dev structure.
4417  * @param[in, out] matcher
4418  *   Pointer to flow matcher.
4419  * @parm[in, out] dev_flow
4420  *   Pointer to the dev_flow.
4421  * @param[out] error
4422  *   pointer to error structure.
4423  *
4424  * @return
4425  *   0 on success otherwise -errno and errno is set.
4426  */
4427 static int
4428 flow_dv_matcher_register(struct rte_eth_dev *dev,
4429                          struct mlx5_flow_dv_matcher *matcher,
4430                          struct mlx5_flow *dev_flow,
4431                          struct rte_flow_error *error)
4432 {
4433         struct mlx5_priv *priv = dev->data->dev_private;
4434         struct mlx5_ibv_shared *sh = priv->sh;
4435         struct mlx5_flow_dv_matcher *cache_matcher;
4436         struct mlx5dv_flow_matcher_attr dv_attr = {
4437                 .type = IBV_FLOW_ATTR_NORMAL,
4438                 .match_mask = (void *)&matcher->mask,
4439         };
4440         struct mlx5_flow_tbl_resource *tbl = NULL;
4441
4442         /* Lookup from cache. */
4443         LIST_FOREACH(cache_matcher, &sh->matchers, next) {
4444                 if (matcher->crc == cache_matcher->crc &&
4445                     matcher->priority == cache_matcher->priority &&
4446                     matcher->egress == cache_matcher->egress &&
4447                     matcher->group == cache_matcher->group &&
4448                     matcher->transfer == cache_matcher->transfer &&
4449                     !memcmp((const void *)matcher->mask.buf,
4450                             (const void *)cache_matcher->mask.buf,
4451                             cache_matcher->mask.size)) {
4452                         DRV_LOG(DEBUG,
4453                                 "priority %hd use %s matcher %p: refcnt %d++",
4454                                 cache_matcher->priority,
4455                                 cache_matcher->egress ? "tx" : "rx",
4456                                 (void *)cache_matcher,
4457                                 rte_atomic32_read(&cache_matcher->refcnt));
4458                         rte_atomic32_inc(&cache_matcher->refcnt);
4459                         dev_flow->dv.matcher = cache_matcher;
4460                         return 0;
4461                 }
4462         }
4463         /* Register new matcher. */
4464         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
4465         if (!cache_matcher)
4466                 return rte_flow_error_set(error, ENOMEM,
4467                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4468                                           "cannot allocate matcher memory");
4469         tbl = flow_dv_tbl_resource_get(dev, matcher->group * MLX5_GROUP_FACTOR,
4470                                        matcher->egress, matcher->transfer,
4471                                        error);
4472         if (!tbl) {
4473                 rte_free(cache_matcher);
4474                 return rte_flow_error_set(error, ENOMEM,
4475                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4476                                           NULL, "cannot create table");
4477         }
4478         *cache_matcher = *matcher;
4479         dv_attr.match_criteria_enable =
4480                 flow_dv_matcher_enable(cache_matcher->mask.buf);
4481         dv_attr.priority = matcher->priority;
4482         if (matcher->egress)
4483                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
4484         cache_matcher->matcher_object =
4485                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
4486         if (!cache_matcher->matcher_object) {
4487                 rte_free(cache_matcher);
4488 #ifdef HAVE_MLX5DV_DR
4489                 flow_dv_tbl_resource_release(tbl);
4490 #endif
4491                 return rte_flow_error_set(error, ENOMEM,
4492                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4493                                           NULL, "cannot create matcher");
4494         }
4495         rte_atomic32_inc(&cache_matcher->refcnt);
4496         LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
4497         dev_flow->dv.matcher = cache_matcher;
4498         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
4499                 cache_matcher->priority,
4500                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
4501                 rte_atomic32_read(&cache_matcher->refcnt));
4502         rte_atomic32_inc(&tbl->refcnt);
4503         return 0;
4504 }
4505
4506 /**
4507  * Find existing tag resource or create and register a new one.
4508  *
4509  * @param dev[in, out]
4510  *   Pointer to rte_eth_dev structure.
4511  * @param[in, out] resource
4512  *   Pointer to tag resource.
4513  * @parm[in, out] dev_flow
4514  *   Pointer to the dev_flow.
4515  * @param[out] error
4516  *   pointer to error structure.
4517  *
4518  * @return
4519  *   0 on success otherwise -errno and errno is set.
4520  */
4521 static int
4522 flow_dv_tag_resource_register
4523                         (struct rte_eth_dev *dev,
4524                          struct mlx5_flow_dv_tag_resource *resource,
4525                          struct mlx5_flow *dev_flow,
4526                          struct rte_flow_error *error)
4527 {
4528         struct mlx5_priv *priv = dev->data->dev_private;
4529         struct mlx5_ibv_shared *sh = priv->sh;
4530         struct mlx5_flow_dv_tag_resource *cache_resource;
4531
4532         /* Lookup a matching resource from cache. */
4533         LIST_FOREACH(cache_resource, &sh->tags, next) {
4534                 if (resource->tag == cache_resource->tag) {
4535                         DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
4536                                 (void *)cache_resource,
4537                                 rte_atomic32_read(&cache_resource->refcnt));
4538                         rte_atomic32_inc(&cache_resource->refcnt);
4539                         dev_flow->flow->tag_resource = cache_resource;
4540                         return 0;
4541                 }
4542         }
4543         /* Register new  resource. */
4544         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
4545         if (!cache_resource)
4546                 return rte_flow_error_set(error, ENOMEM,
4547                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4548                                           "cannot allocate resource memory");
4549         *cache_resource = *resource;
4550         cache_resource->action = mlx5_glue->dv_create_flow_action_tag
4551                 (resource->tag);
4552         if (!cache_resource->action) {
4553                 rte_free(cache_resource);
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4556                                           NULL, "cannot create action");
4557         }
4558         rte_atomic32_init(&cache_resource->refcnt);
4559         rte_atomic32_inc(&cache_resource->refcnt);
4560         LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
4561         dev_flow->flow->tag_resource = cache_resource;
4562         DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
4563                 (void *)cache_resource,
4564                 rte_atomic32_read(&cache_resource->refcnt));
4565         return 0;
4566 }
4567
4568 /**
4569  * Release the tag.
4570  *
4571  * @param dev
4572  *   Pointer to Ethernet device.
4573  * @param flow
4574  *   Pointer to mlx5_flow.
4575  *
4576  * @return
4577  *   1 while a reference on it exists, 0 when freed.
4578  */
4579 static int
4580 flow_dv_tag_release(struct rte_eth_dev *dev,
4581                     struct mlx5_flow_dv_tag_resource *tag)
4582 {
4583         assert(tag);
4584         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
4585                 dev->data->port_id, (void *)tag,
4586                 rte_atomic32_read(&tag->refcnt));
4587         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
4588                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
4589                 LIST_REMOVE(tag, next);
4590                 DRV_LOG(DEBUG, "port %u tag %p: removed",
4591                         dev->data->port_id, (void *)tag);
4592                 rte_free(tag);
4593                 return 0;
4594         }
4595         return 1;
4596 }
4597
4598 /**
4599  * Translate port ID action to vport.
4600  *
4601  * @param[in] dev
4602  *   Pointer to rte_eth_dev structure.
4603  * @param[in] action
4604  *   Pointer to the port ID action.
4605  * @param[out] dst_port_id
4606  *   The target port ID.
4607  * @param[out] error
4608  *   Pointer to the error structure.
4609  *
4610  * @return
4611  *   0 on success, a negative errno value otherwise and rte_errno is set.
4612  */
4613 static int
4614 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
4615                                  const struct rte_flow_action *action,
4616                                  uint32_t *dst_port_id,
4617                                  struct rte_flow_error *error)
4618 {
4619         uint32_t port;
4620         uint16_t port_id;
4621         int ret;
4622         const struct rte_flow_action_port_id *conf =
4623                         (const struct rte_flow_action_port_id *)action->conf;
4624
4625         port = conf->original ? dev->data->port_id : conf->id;
4626         ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
4627         if (ret)
4628                 return rte_flow_error_set(error, -ret,
4629                                           RTE_FLOW_ERROR_TYPE_ACTION,
4630                                           NULL,
4631                                           "No eswitch info was found for port");
4632         *dst_port_id = port_id;
4633         return 0;
4634 }
4635
4636 /**
4637  * Fill the flow with DV spec.
4638  *
4639  * @param[in] dev
4640  *   Pointer to rte_eth_dev structure.
4641  * @param[in, out] dev_flow
4642  *   Pointer to the sub flow.
4643  * @param[in] attr
4644  *   Pointer to the flow attributes.
4645  * @param[in] items
4646  *   Pointer to the list of items.
4647  * @param[in] actions
4648  *   Pointer to the list of actions.
4649  * @param[out] error
4650  *   Pointer to the error structure.
4651  *
4652  * @return
4653  *   0 on success, a negative errno value otherwise and rte_errno is set.
4654  */
4655 static int
4656 flow_dv_translate(struct rte_eth_dev *dev,
4657                   struct mlx5_flow *dev_flow,
4658                   const struct rte_flow_attr *attr,
4659                   const struct rte_flow_item items[],
4660                   const struct rte_flow_action actions[],
4661                   struct rte_flow_error *error)
4662 {
4663         struct mlx5_priv *priv = dev->data->dev_private;
4664         struct rte_flow *flow = dev_flow->flow;
4665         uint64_t item_flags = 0;
4666         uint64_t last_item = 0;
4667         uint64_t action_flags = 0;
4668         uint64_t priority = attr->priority;
4669         struct mlx5_flow_dv_matcher matcher = {
4670                 .mask = {
4671                         .size = sizeof(matcher.mask.buf),
4672                 },
4673         };
4674         int actions_n = 0;
4675         bool actions_end = false;
4676         struct mlx5_flow_dv_modify_hdr_resource res = {
4677                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4678                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
4679         };
4680         union flow_dv_attr flow_attr = { .attr = 0 };
4681         struct mlx5_flow_dv_tag_resource tag_resource;
4682         uint32_t modify_action_position = UINT32_MAX;
4683         void *match_mask = matcher.mask.buf;
4684         void *match_value = dev_flow->dv.value.buf;
4685         uint8_t next_protocol = 0xff;
4686
4687         flow->group = attr->group;
4688         if (attr->transfer)
4689                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4690         if (priority == MLX5_FLOW_PRIO_RSVD)
4691                 priority = priv->config.flow_prio - 1;
4692         for (; !actions_end ; actions++) {
4693                 const struct rte_flow_action_queue *queue;
4694                 const struct rte_flow_action_rss *rss;
4695                 const struct rte_flow_action *action = actions;
4696                 const struct rte_flow_action_count *count = action->conf;
4697                 const uint8_t *rss_key;
4698                 const struct rte_flow_action_jump *jump_data;
4699                 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
4700                 struct mlx5_flow_tbl_resource *tbl;
4701                 uint32_t port_id = 0;
4702                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
4703
4704                 switch (actions->type) {
4705                 case RTE_FLOW_ACTION_TYPE_VOID:
4706                         break;
4707                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4708                         if (flow_dv_translate_action_port_id(dev, action,
4709                                                              &port_id, error))
4710                                 return -rte_errno;
4711                         port_id_resource.port_id = port_id;
4712                         if (flow_dv_port_id_action_resource_register
4713                             (dev, &port_id_resource, dev_flow, error))
4714                                 return -rte_errno;
4715                         dev_flow->dv.actions[actions_n++] =
4716                                 dev_flow->dv.port_id_action->action;
4717                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4718                         break;
4719                 case RTE_FLOW_ACTION_TYPE_FLAG:
4720                         tag_resource.tag =
4721                                 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
4722                         if (!flow->tag_resource)
4723                                 if (flow_dv_tag_resource_register
4724                                     (dev, &tag_resource, dev_flow, error))
4725                                         return errno;
4726                         dev_flow->dv.actions[actions_n++] =
4727                                 flow->tag_resource->action;
4728                         action_flags |= MLX5_FLOW_ACTION_FLAG;
4729                         break;
4730                 case RTE_FLOW_ACTION_TYPE_MARK:
4731                         tag_resource.tag = mlx5_flow_mark_set
4732                               (((const struct rte_flow_action_mark *)
4733                                (actions->conf))->id);
4734                         if (!flow->tag_resource)
4735                                 if (flow_dv_tag_resource_register
4736                                     (dev, &tag_resource, dev_flow, error))
4737                                         return errno;
4738                         dev_flow->dv.actions[actions_n++] =
4739                                 flow->tag_resource->action;
4740                         action_flags |= MLX5_FLOW_ACTION_MARK;
4741                         break;
4742                 case RTE_FLOW_ACTION_TYPE_DROP:
4743                         action_flags |= MLX5_FLOW_ACTION_DROP;
4744                         break;
4745                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4746                         queue = actions->conf;
4747                         flow->rss.queue_num = 1;
4748                         (*flow->queue)[0] = queue->index;
4749                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
4750                         break;
4751                 case RTE_FLOW_ACTION_TYPE_RSS:
4752                         rss = actions->conf;
4753                         if (flow->queue)
4754                                 memcpy((*flow->queue), rss->queue,
4755                                        rss->queue_num * sizeof(uint16_t));
4756                         flow->rss.queue_num = rss->queue_num;
4757                         /* NULL RSS key indicates default RSS key. */
4758                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
4759                         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
4760                         /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
4761                         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
4762                         flow->rss.level = rss->level;
4763                         action_flags |= MLX5_FLOW_ACTION_RSS;
4764                         break;
4765                 case RTE_FLOW_ACTION_TYPE_COUNT:
4766                         if (!priv->config.devx) {
4767                                 rte_errno = ENOTSUP;
4768                                 goto cnt_err;
4769                         }
4770                         flow->counter = flow_dv_counter_alloc(dev,
4771                                                               count->shared,
4772                                                               count->id,
4773                                                               attr->group);
4774                         if (flow->counter == NULL)
4775                                 goto cnt_err;
4776                         dev_flow->dv.actions[actions_n++] =
4777                                 flow->counter->action;
4778                         action_flags |= MLX5_FLOW_ACTION_COUNT;
4779                         break;
4780 cnt_err:
4781                         if (rte_errno == ENOTSUP)
4782                                 return rte_flow_error_set
4783                                               (error, ENOTSUP,
4784                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4785                                                NULL,
4786                                                "count action not supported");
4787                         else
4788                                 return rte_flow_error_set
4789                                                 (error, rte_errno,
4790                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4791                                                  action,
4792                                                  "cannot create counter"
4793                                                   " object.");
4794                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4795                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4796                         if (flow_dv_create_action_l2_encap(dev, actions,
4797                                                            dev_flow,
4798                                                            attr->transfer,
4799                                                            error))
4800                                 return -rte_errno;
4801                         dev_flow->dv.actions[actions_n++] =
4802                                 dev_flow->dv.encap_decap->verbs_action;
4803                         action_flags |= actions->type ==
4804                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
4805                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
4806                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
4807                         break;
4808                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4809                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4810                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
4811                                                            attr->transfer,
4812                                                            error))
4813                                 return -rte_errno;
4814                         dev_flow->dv.actions[actions_n++] =
4815                                 dev_flow->dv.encap_decap->verbs_action;
4816                         action_flags |= actions->type ==
4817                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
4818                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
4819                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
4820                         break;
4821                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4822                         /* Handle encap with preceding decap. */
4823                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
4824                                 if (flow_dv_create_action_raw_encap
4825                                         (dev, actions, dev_flow, attr, error))
4826                                         return -rte_errno;
4827                                 dev_flow->dv.actions[actions_n++] =
4828                                         dev_flow->dv.encap_decap->verbs_action;
4829                         } else {
4830                                 /* Handle encap without preceding decap. */
4831                                 if (flow_dv_create_action_l2_encap
4832                                     (dev, actions, dev_flow, attr->transfer,
4833                                      error))
4834                                         return -rte_errno;
4835                                 dev_flow->dv.actions[actions_n++] =
4836                                         dev_flow->dv.encap_decap->verbs_action;
4837                         }
4838                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
4839                         break;
4840                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4841                         /* Check if this decap is followed by encap. */
4842                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
4843                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
4844                                action++) {
4845                         }
4846                         /* Handle decap only if it isn't followed by encap. */
4847                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4848                                 if (flow_dv_create_action_l2_decap
4849                                     (dev, dev_flow, attr->transfer, error))
4850                                         return -rte_errno;
4851                                 dev_flow->dv.actions[actions_n++] =
4852                                         dev_flow->dv.encap_decap->verbs_action;
4853                         }
4854                         /* If decap is followed by encap, handle it at encap. */
4855                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
4856                         break;
4857                 case RTE_FLOW_ACTION_TYPE_JUMP:
4858                         jump_data = action->conf;
4859                         tbl = flow_dv_tbl_resource_get(dev, jump_data->group *
4860                                                        MLX5_GROUP_FACTOR,
4861                                                        attr->egress,
4862                                                        attr->transfer, error);
4863                         if (!tbl)
4864                                 return rte_flow_error_set
4865                                                 (error, errno,
4866                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4867                                                  NULL,
4868                                                  "cannot create jump action.");
4869                         jump_tbl_resource.tbl = tbl;
4870                         if (flow_dv_jump_tbl_resource_register
4871                             (dev, &jump_tbl_resource, dev_flow, error)) {
4872                                 flow_dv_tbl_resource_release(tbl);
4873                                 return rte_flow_error_set
4874                                                 (error, errno,
4875                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4876                                                  NULL,
4877                                                  "cannot create jump action.");
4878                         }
4879                         dev_flow->dv.actions[actions_n++] =
4880                                 dev_flow->dv.jump->action;
4881                         action_flags |= MLX5_FLOW_ACTION_JUMP;
4882                         break;
4883                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4884                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4885                         if (flow_dv_convert_action_modify_mac(&res, actions,
4886                                                               error))
4887                                 return -rte_errno;
4888                         action_flags |= actions->type ==
4889                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
4890                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
4891                                         MLX5_FLOW_ACTION_SET_MAC_DST;
4892                         break;
4893                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4894                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4895                         if (flow_dv_convert_action_modify_ipv4(&res, actions,
4896                                                                error))
4897                                 return -rte_errno;
4898                         action_flags |= actions->type ==
4899                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
4900                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
4901                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
4902                         break;
4903                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4904                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4905                         if (flow_dv_convert_action_modify_ipv6(&res, actions,
4906                                                                error))
4907                                 return -rte_errno;
4908                         action_flags |= actions->type ==
4909                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
4910                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
4911                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
4912                         break;
4913                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4914                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4915                         if (flow_dv_convert_action_modify_tp(&res, actions,
4916                                                              items, &flow_attr,
4917                                                              error))
4918                                 return -rte_errno;
4919                         action_flags |= actions->type ==
4920                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
4921                                         MLX5_FLOW_ACTION_SET_TP_SRC :
4922                                         MLX5_FLOW_ACTION_SET_TP_DST;
4923                         break;
4924                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4925                         if (flow_dv_convert_action_modify_dec_ttl(&res, items,
4926                                                                   &flow_attr,
4927                                                                   error))
4928                                 return -rte_errno;
4929                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
4930                         break;
4931                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
4932                         if (flow_dv_convert_action_modify_ttl(&res, actions,
4933                                                              items, &flow_attr,
4934                                                              error))
4935                                 return -rte_errno;
4936                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
4937                         break;
4938                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4939                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4940                         if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
4941                                                                   error))
4942                                 return -rte_errno;
4943                         action_flags |= actions->type ==
4944                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
4945                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
4946                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
4947                         break;
4948
4949                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4950                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4951                         if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
4952                                                                   error))
4953                                 return -rte_errno;
4954                         action_flags |= actions->type ==
4955                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
4956                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
4957                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
4958                         break;
4959                 case RTE_FLOW_ACTION_TYPE_END:
4960                         actions_end = true;
4961                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
4962                                 /* create modify action if needed. */
4963                                 if (flow_dv_modify_hdr_resource_register
4964                                                                 (dev, &res,
4965                                                                  dev_flow,
4966                                                                  error))
4967                                         return -rte_errno;
4968                                 dev_flow->dv.actions[modify_action_position] =
4969                                         dev_flow->dv.modify_hdr->verbs_action;
4970                         }
4971                         break;
4972                 default:
4973                         break;
4974                 }
4975                 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
4976                     modify_action_position == UINT32_MAX)
4977                         modify_action_position = actions_n++;
4978         }
4979         dev_flow->dv.actions_n = actions_n;
4980         flow->actions = action_flags;
4981         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4982                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4983
4984                 switch (items->type) {
4985                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4986                         flow_dv_translate_item_port_id(dev, match_mask,
4987                                                        match_value, items);
4988                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4989                         break;
4990                 case RTE_FLOW_ITEM_TYPE_ETH:
4991                         flow_dv_translate_item_eth(match_mask, match_value,
4992                                                    items, tunnel);
4993                         matcher.priority = MLX5_PRIORITY_MAP_L2;
4994                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4995                                              MLX5_FLOW_LAYER_OUTER_L2;
4996                         break;
4997                 case RTE_FLOW_ITEM_TYPE_VLAN:
4998                         flow_dv_translate_item_vlan(dev_flow,
4999                                                     match_mask, match_value,
5000                                                     items, tunnel);
5001                         matcher.priority = MLX5_PRIORITY_MAP_L2;
5002                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
5003                                               MLX5_FLOW_LAYER_INNER_VLAN) :
5004                                              (MLX5_FLOW_LAYER_OUTER_L2 |
5005                                               MLX5_FLOW_LAYER_OUTER_VLAN);
5006                         break;
5007                 case RTE_FLOW_ITEM_TYPE_IPV4:
5008                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5009                                                   &item_flags, &tunnel);
5010                         flow_dv_translate_item_ipv4(match_mask, match_value,
5011                                                     items, tunnel, attr->group);
5012                         matcher.priority = MLX5_PRIORITY_MAP_L3;
5013                         dev_flow->dv.hash_fields |=
5014                                 mlx5_flow_hashfields_adjust
5015                                         (dev_flow, tunnel,
5016                                          MLX5_IPV4_LAYER_TYPES,
5017                                          MLX5_IPV4_IBV_RX_HASH);
5018                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5019                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5020                         if (items->mask != NULL &&
5021                             ((const struct rte_flow_item_ipv4 *)
5022                              items->mask)->hdr.next_proto_id) {
5023                                 next_protocol =
5024                                         ((const struct rte_flow_item_ipv4 *)
5025                                          (items->spec))->hdr.next_proto_id;
5026                                 next_protocol &=
5027                                         ((const struct rte_flow_item_ipv4 *)
5028                                          (items->mask))->hdr.next_proto_id;
5029                         } else {
5030                                 /* Reset for inner layer. */
5031                                 next_protocol = 0xff;
5032                         }
5033                         break;
5034                 case RTE_FLOW_ITEM_TYPE_IPV6:
5035                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5036                                                   &item_flags, &tunnel);
5037                         flow_dv_translate_item_ipv6(match_mask, match_value,
5038                                                     items, tunnel, attr->group);
5039                         matcher.priority = MLX5_PRIORITY_MAP_L3;
5040                         dev_flow->dv.hash_fields |=
5041                                 mlx5_flow_hashfields_adjust
5042                                         (dev_flow, tunnel,
5043                                          MLX5_IPV6_LAYER_TYPES,
5044                                          MLX5_IPV6_IBV_RX_HASH);
5045                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5046                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5047                         if (items->mask != NULL &&
5048                             ((const struct rte_flow_item_ipv6 *)
5049                              items->mask)->hdr.proto) {
5050                                 next_protocol =
5051                                         ((const struct rte_flow_item_ipv6 *)
5052                                          items->spec)->hdr.proto;
5053                                 next_protocol &=
5054                                         ((const struct rte_flow_item_ipv6 *)
5055                                          items->mask)->hdr.proto;
5056                         } else {
5057                                 /* Reset for inner layer. */
5058                                 next_protocol = 0xff;
5059                         }
5060                         break;
5061                 case RTE_FLOW_ITEM_TYPE_TCP:
5062                         flow_dv_translate_item_tcp(match_mask, match_value,
5063                                                    items, tunnel);
5064                         matcher.priority = MLX5_PRIORITY_MAP_L4;
5065                         dev_flow->dv.hash_fields |=
5066                                 mlx5_flow_hashfields_adjust
5067                                         (dev_flow, tunnel, ETH_RSS_TCP,
5068                                          IBV_RX_HASH_SRC_PORT_TCP |
5069                                          IBV_RX_HASH_DST_PORT_TCP);
5070                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5071                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5072                         break;
5073                 case RTE_FLOW_ITEM_TYPE_UDP:
5074                         flow_dv_translate_item_udp(match_mask, match_value,
5075                                                    items, tunnel);
5076                         matcher.priority = MLX5_PRIORITY_MAP_L4;
5077                         dev_flow->dv.hash_fields |=
5078                                 mlx5_flow_hashfields_adjust
5079                                         (dev_flow, tunnel, ETH_RSS_UDP,
5080                                          IBV_RX_HASH_SRC_PORT_UDP |
5081                                          IBV_RX_HASH_DST_PORT_UDP);
5082                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5083                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5084                         break;
5085                 case RTE_FLOW_ITEM_TYPE_GRE:
5086                         flow_dv_translate_item_gre(match_mask, match_value,
5087                                                    items, tunnel);
5088                         last_item = MLX5_FLOW_LAYER_GRE;
5089                         break;
5090                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5091                         flow_dv_translate_item_gre_key(match_mask,
5092                                                        match_value, items);
5093                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5094                         break;
5095                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5096                         flow_dv_translate_item_nvgre(match_mask, match_value,
5097                                                      items, tunnel);
5098                         last_item = MLX5_FLOW_LAYER_GRE;
5099                         break;
5100                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5101                         flow_dv_translate_item_vxlan(match_mask, match_value,
5102                                                      items, tunnel);
5103                         last_item = MLX5_FLOW_LAYER_VXLAN;
5104                         break;
5105                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5106                         flow_dv_translate_item_vxlan(match_mask, match_value,
5107                                                      items, tunnel);
5108                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5109                         break;
5110                 case RTE_FLOW_ITEM_TYPE_MPLS:
5111                         flow_dv_translate_item_mpls(match_mask, match_value,
5112                                                     items, last_item, tunnel);
5113                         last_item = MLX5_FLOW_LAYER_MPLS;
5114                         break;
5115                 case RTE_FLOW_ITEM_TYPE_META:
5116                         flow_dv_translate_item_meta(match_mask, match_value,
5117                                                     items);
5118                         last_item = MLX5_FLOW_ITEM_METADATA;
5119                         break;
5120                 case RTE_FLOW_ITEM_TYPE_ICMP:
5121                         flow_dv_translate_item_icmp(match_mask, match_value,
5122                                                     items, tunnel);
5123                         last_item = MLX5_FLOW_LAYER_ICMP;
5124                         break;
5125                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5126                         flow_dv_translate_item_icmp6(match_mask, match_value,
5127                                                       items, tunnel);
5128                         last_item = MLX5_FLOW_LAYER_ICMP6;
5129                         break;
5130                 default:
5131                         break;
5132                 }
5133                 item_flags |= last_item;
5134         }
5135         /*
5136          * In case of ingress traffic when E-Switch mode is enabled,
5137          * we have two cases where we need to set the source port manually.
5138          * The first one, is in case of Nic steering rule, and the second is
5139          * E-Switch rule where no port_id item was found. In both cases
5140          * the source port is set according the current port in use.
5141          */
5142         if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
5143             (priv->representor || priv->master)) {
5144                 if (flow_dv_translate_item_port_id(dev, match_mask,
5145                                                    match_value, NULL))
5146                         return -rte_errno;
5147         }
5148         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
5149                                          dev_flow->dv.value.buf));
5150         dev_flow->layers = item_flags;
5151         /* Register matcher. */
5152         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
5153                                     matcher.mask.size);
5154         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
5155                                                      matcher.priority);
5156         matcher.egress = attr->egress;
5157         matcher.group = attr->group;
5158         matcher.transfer = attr->transfer;
5159         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
5160                 return -rte_errno;
5161         return 0;
5162 }
5163
5164 /**
5165  * Apply the flow to the NIC.
5166  *
5167  * @param[in] dev
5168  *   Pointer to the Ethernet device structure.
5169  * @param[in, out] flow
5170  *   Pointer to flow structure.
5171  * @param[out] error
5172  *   Pointer to error structure.
5173  *
5174  * @return
5175  *   0 on success, a negative errno value otherwise and rte_errno is set.
5176  */
5177 static int
5178 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5179               struct rte_flow_error *error)
5180 {
5181         struct mlx5_flow_dv *dv;
5182         struct mlx5_flow *dev_flow;
5183         struct mlx5_priv *priv = dev->data->dev_private;
5184         int n;
5185         int err;
5186
5187         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5188                 dv = &dev_flow->dv;
5189                 n = dv->actions_n;
5190                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
5191                         if (flow->transfer) {
5192                                 dv->actions[n++] = priv->sh->esw_drop_action;
5193                         } else {
5194                                 dv->hrxq = mlx5_hrxq_drop_new(dev);
5195                                 if (!dv->hrxq) {
5196                                         rte_flow_error_set
5197                                                 (error, errno,
5198                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5199                                                  NULL,
5200                                                  "cannot get drop hash queue");
5201                                         goto error;
5202                                 }
5203                                 dv->actions[n++] = dv->hrxq->action;
5204                         }
5205                 } else if (flow->actions &
5206                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
5207                         struct mlx5_hrxq *hrxq;
5208
5209                         hrxq = mlx5_hrxq_get(dev, flow->key,
5210                                              MLX5_RSS_HASH_KEY_LEN,
5211                                              dv->hash_fields,
5212                                              (*flow->queue),
5213                                              flow->rss.queue_num);
5214                         if (!hrxq) {
5215                                 hrxq = mlx5_hrxq_new
5216                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
5217                                          dv->hash_fields, (*flow->queue),
5218                                          flow->rss.queue_num,
5219                                          !!(dev_flow->layers &
5220                                             MLX5_FLOW_LAYER_TUNNEL));
5221                         }
5222                         if (!hrxq) {
5223                                 rte_flow_error_set
5224                                         (error, rte_errno,
5225                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5226                                          "cannot get hash queue");
5227                                 goto error;
5228                         }
5229                         dv->hrxq = hrxq;
5230                         dv->actions[n++] = dv->hrxq->action;
5231                 }
5232                 dv->flow =
5233                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
5234                                                   (void *)&dv->value, n,
5235                                                   dv->actions);
5236                 if (!dv->flow) {
5237                         rte_flow_error_set(error, errno,
5238                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5239                                            NULL,
5240                                            "hardware refuses to create flow");
5241                         goto error;
5242                 }
5243                 if (priv->vmwa_context &&
5244                     dev_flow->dv.vf_vlan.tag &&
5245                     !dev_flow->dv.vf_vlan.created) {
5246                         /*
5247                          * The rule contains the VLAN pattern.
5248                          * For VF we are going to create VLAN
5249                          * interface to make hypervisor set correct
5250                          * e-Switch vport context.
5251                          */
5252                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
5253                 }
5254         }
5255         return 0;
5256 error:
5257         err = rte_errno; /* Save rte_errno before cleanup. */
5258         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5259                 struct mlx5_flow_dv *dv = &dev_flow->dv;
5260                 if (dv->hrxq) {
5261                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
5262                                 mlx5_hrxq_drop_release(dev);
5263                         else
5264                                 mlx5_hrxq_release(dev, dv->hrxq);
5265                         dv->hrxq = NULL;
5266                 }
5267                 if (dev_flow->dv.vf_vlan.tag &&
5268                     dev_flow->dv.vf_vlan.created)
5269                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5270         }
5271         rte_errno = err; /* Restore rte_errno. */
5272         return -rte_errno;
5273 }
5274
5275 /**
5276  * Release the flow matcher.
5277  *
5278  * @param dev
5279  *   Pointer to Ethernet device.
5280  * @param flow
5281  *   Pointer to mlx5_flow.
5282  *
5283  * @return
5284  *   1 while a reference on it exists, 0 when freed.
5285  */
5286 static int
5287 flow_dv_matcher_release(struct rte_eth_dev *dev,
5288                         struct mlx5_flow *flow)
5289 {
5290         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
5291         struct mlx5_priv *priv = dev->data->dev_private;
5292         struct mlx5_ibv_shared *sh = priv->sh;
5293         struct mlx5_flow_tbl_resource *tbl;
5294
5295         assert(matcher->matcher_object);
5296         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
5297                 dev->data->port_id, (void *)matcher,
5298                 rte_atomic32_read(&matcher->refcnt));
5299         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
5300                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
5301                            (matcher->matcher_object));
5302                 LIST_REMOVE(matcher, next);
5303                 if (matcher->egress)
5304                         tbl = &sh->tx_tbl[matcher->group];
5305                 else
5306                         tbl = &sh->rx_tbl[matcher->group];
5307                 flow_dv_tbl_resource_release(tbl);
5308                 rte_free(matcher);
5309                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
5310                         dev->data->port_id, (void *)matcher);
5311                 return 0;
5312         }
5313         return 1;
5314 }
5315
5316 /**
5317  * Release an encap/decap resource.
5318  *
5319  * @param flow
5320  *   Pointer to mlx5_flow.
5321  *
5322  * @return
5323  *   1 while a reference on it exists, 0 when freed.
5324  */
5325 static int
5326 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
5327 {
5328         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
5329                                                 flow->dv.encap_decap;
5330
5331         assert(cache_resource->verbs_action);
5332         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
5333                 (void *)cache_resource,
5334                 rte_atomic32_read(&cache_resource->refcnt));
5335         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5336                 claim_zero(mlx5_glue->destroy_flow_action
5337                                 (cache_resource->verbs_action));
5338                 LIST_REMOVE(cache_resource, next);
5339                 rte_free(cache_resource);
5340                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
5341                         (void *)cache_resource);
5342                 return 0;
5343         }
5344         return 1;
5345 }
5346
5347 /**
5348  * Release an jump to table action resource.
5349  *
5350  * @param flow
5351  *   Pointer to mlx5_flow.
5352  *
5353  * @return
5354  *   1 while a reference on it exists, 0 when freed.
5355  */
5356 static int
5357 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
5358 {
5359         struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
5360                                                 flow->dv.jump;
5361
5362         assert(cache_resource->action);
5363         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
5364                 (void *)cache_resource,
5365                 rte_atomic32_read(&cache_resource->refcnt));
5366         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5367                 claim_zero(mlx5_glue->destroy_flow_action
5368                                 (cache_resource->action));
5369                 LIST_REMOVE(cache_resource, next);
5370                 flow_dv_tbl_resource_release(cache_resource->tbl);
5371                 rte_free(cache_resource);
5372                 DRV_LOG(DEBUG, "jump table resource %p: removed",
5373                         (void *)cache_resource);
5374                 return 0;
5375         }
5376         return 1;
5377 }
5378
5379 /**
5380  * Release a modify-header resource.
5381  *
5382  * @param flow
5383  *   Pointer to mlx5_flow.
5384  *
5385  * @return
5386  *   1 while a reference on it exists, 0 when freed.
5387  */
5388 static int
5389 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
5390 {
5391         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
5392                                                 flow->dv.modify_hdr;
5393
5394         assert(cache_resource->verbs_action);
5395         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
5396                 (void *)cache_resource,
5397                 rte_atomic32_read(&cache_resource->refcnt));
5398         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5399                 claim_zero(mlx5_glue->destroy_flow_action
5400                                 (cache_resource->verbs_action));
5401                 LIST_REMOVE(cache_resource, next);
5402                 rte_free(cache_resource);
5403                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
5404                         (void *)cache_resource);
5405                 return 0;
5406         }
5407         return 1;
5408 }
5409
5410 /**
5411  * Release port ID action resource.
5412  *
5413  * @param flow
5414  *   Pointer to mlx5_flow.
5415  *
5416  * @return
5417  *   1 while a reference on it exists, 0 when freed.
5418  */
5419 static int
5420 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
5421 {
5422         struct mlx5_flow_dv_port_id_action_resource *cache_resource =
5423                 flow->dv.port_id_action;
5424
5425         assert(cache_resource->action);
5426         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
5427                 (void *)cache_resource,
5428                 rte_atomic32_read(&cache_resource->refcnt));
5429         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5430                 claim_zero(mlx5_glue->destroy_flow_action
5431                                 (cache_resource->action));
5432                 LIST_REMOVE(cache_resource, next);
5433                 rte_free(cache_resource);
5434                 DRV_LOG(DEBUG, "port id action resource %p: removed",
5435                         (void *)cache_resource);
5436                 return 0;
5437         }
5438         return 1;
5439 }
5440
5441 /**
5442  * Remove the flow from the NIC but keeps it in memory.
5443  *
5444  * @param[in] dev
5445  *   Pointer to Ethernet device.
5446  * @param[in, out] flow
5447  *   Pointer to flow structure.
5448  */
5449 static void
5450 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5451 {
5452         struct mlx5_flow_dv *dv;
5453         struct mlx5_flow *dev_flow;
5454
5455         if (!flow)
5456                 return;
5457         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5458                 dv = &dev_flow->dv;
5459                 if (dv->flow) {
5460                         claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
5461                         dv->flow = NULL;
5462                 }
5463                 if (dv->hrxq) {
5464                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
5465                                 mlx5_hrxq_drop_release(dev);
5466                         else
5467                                 mlx5_hrxq_release(dev, dv->hrxq);
5468                         dv->hrxq = NULL;
5469                 }
5470                 if (dev_flow->dv.vf_vlan.tag &&
5471                     dev_flow->dv.vf_vlan.created)
5472                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5473         }
5474 }
5475
5476 /**
5477  * Remove the flow from the NIC and the memory.
5478  *
5479  * @param[in] dev
5480  *   Pointer to the Ethernet device structure.
5481  * @param[in, out] flow
5482  *   Pointer to flow structure.
5483  */
5484 static void
5485 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5486 {
5487         struct mlx5_flow *dev_flow;
5488
5489         if (!flow)
5490                 return;
5491         flow_dv_remove(dev, flow);
5492         if (flow->counter) {
5493                 flow_dv_counter_release(dev, flow->counter);
5494                 flow->counter = NULL;
5495         }
5496         if (flow->tag_resource) {
5497                 flow_dv_tag_release(dev, flow->tag_resource);
5498                 flow->tag_resource = NULL;
5499         }
5500         while (!LIST_EMPTY(&flow->dev_flows)) {
5501                 dev_flow = LIST_FIRST(&flow->dev_flows);
5502                 LIST_REMOVE(dev_flow, next);
5503                 if (dev_flow->dv.matcher)
5504                         flow_dv_matcher_release(dev, dev_flow);
5505                 if (dev_flow->dv.encap_decap)
5506                         flow_dv_encap_decap_resource_release(dev_flow);
5507                 if (dev_flow->dv.modify_hdr)
5508                         flow_dv_modify_hdr_resource_release(dev_flow);
5509                 if (dev_flow->dv.jump)
5510                         flow_dv_jump_tbl_resource_release(dev_flow);
5511                 if (dev_flow->dv.port_id_action)
5512                         flow_dv_port_id_action_resource_release(dev_flow);
5513                 rte_free(dev_flow);
5514         }
5515 }
5516
5517 /**
5518  * Query a dv flow  rule for its statistics via devx.
5519  *
5520  * @param[in] dev
5521  *   Pointer to Ethernet device.
5522  * @param[in] flow
5523  *   Pointer to the sub flow.
5524  * @param[out] data
5525  *   data retrieved by the query.
5526  * @param[out] error
5527  *   Perform verbose error reporting if not NULL.
5528  *
5529  * @return
5530  *   0 on success, a negative errno value otherwise and rte_errno is set.
5531  */
5532 static int
5533 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
5534                     void *data, struct rte_flow_error *error)
5535 {
5536         struct mlx5_priv *priv = dev->data->dev_private;
5537         struct rte_flow_query_count *qc = data;
5538
5539         if (!priv->config.devx)
5540                 return rte_flow_error_set(error, ENOTSUP,
5541                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5542                                           NULL,
5543                                           "counters are not supported");
5544         if (flow->counter) {
5545                 uint64_t pkts, bytes;
5546                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
5547                                                &bytes);
5548
5549                 if (err)
5550                         return rte_flow_error_set(error, -err,
5551                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5552                                         NULL, "cannot read counters");
5553                 qc->hits_set = 1;
5554                 qc->bytes_set = 1;
5555                 qc->hits = pkts - flow->counter->hits;
5556                 qc->bytes = bytes - flow->counter->bytes;
5557                 if (qc->reset) {
5558                         flow->counter->hits = pkts;
5559                         flow->counter->bytes = bytes;
5560                 }
5561                 return 0;
5562         }
5563         return rte_flow_error_set(error, EINVAL,
5564                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5565                                   NULL,
5566                                   "counters are not available");
5567 }
5568
5569 /**
5570  * Query a flow.
5571  *
5572  * @see rte_flow_query()
5573  * @see rte_flow_ops
5574  */
5575 static int
5576 flow_dv_query(struct rte_eth_dev *dev,
5577               struct rte_flow *flow __rte_unused,
5578               const struct rte_flow_action *actions __rte_unused,
5579               void *data __rte_unused,
5580               struct rte_flow_error *error __rte_unused)
5581 {
5582         int ret = -EINVAL;
5583
5584         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5585                 switch (actions->type) {
5586                 case RTE_FLOW_ACTION_TYPE_VOID:
5587                         break;
5588                 case RTE_FLOW_ACTION_TYPE_COUNT:
5589                         ret = flow_dv_query_count(dev, flow, data, error);
5590                         break;
5591                 default:
5592                         return rte_flow_error_set(error, ENOTSUP,
5593                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5594                                                   actions,
5595                                                   "action not supported");
5596                 }
5597         }
5598         return ret;
5599 }
5600
5601 /*
5602  * Mutex-protected thunk to flow_dv_translate().
5603  */
5604 static int
5605 flow_d_translate(struct rte_eth_dev *dev,
5606                  struct mlx5_flow *dev_flow,
5607                  const struct rte_flow_attr *attr,
5608                  const struct rte_flow_item items[],
5609                  const struct rte_flow_action actions[],
5610                  struct rte_flow_error *error)
5611 {
5612         int ret;
5613
5614         flow_d_shared_lock(dev);
5615         ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
5616         flow_d_shared_unlock(dev);
5617         return ret;
5618 }
5619
5620 /*
5621  * Mutex-protected thunk to flow_dv_apply().
5622  */
5623 static int
5624 flow_d_apply(struct rte_eth_dev *dev,
5625              struct rte_flow *flow,
5626              struct rte_flow_error *error)
5627 {
5628         int ret;
5629
5630         flow_d_shared_lock(dev);
5631         ret = flow_dv_apply(dev, flow, error);
5632         flow_d_shared_unlock(dev);
5633         return ret;
5634 }
5635
5636 /*
5637  * Mutex-protected thunk to flow_dv_remove().
5638  */
5639 static void
5640 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5641 {
5642         flow_d_shared_lock(dev);
5643         flow_dv_remove(dev, flow);
5644         flow_d_shared_unlock(dev);
5645 }
5646
5647 /*
5648  * Mutex-protected thunk to flow_dv_destroy().
5649  */
5650 static void
5651 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5652 {
5653         flow_d_shared_lock(dev);
5654         flow_dv_destroy(dev, flow);
5655         flow_d_shared_unlock(dev);
5656 }
5657
5658 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
5659         .validate = flow_dv_validate,
5660         .prepare = flow_dv_prepare,
5661         .translate = flow_d_translate,
5662         .apply = flow_d_apply,
5663         .remove = flow_d_remove,
5664         .destroy = flow_d_destroy,
5665         .query = flow_dv_query,
5666 };
5667
5668 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */