drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_prm.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 union flow_dv_attr {
  54         struct {
  55                 uint32_t valid:1;
  56                 uint32_t ipv4:1;
  57                 uint32_t ipv6:1;
  58                 uint32_t tcp:1;
  59                 uint32_t udp:1;
  60                 uint32_t reserved:27;
  61         };
  62         uint32_t attr;
  63 };
  64
  65 #define MLX5_FLOW_IPV4_LRO (1 << 0)
  66 #define MLX5_FLOW_IPV6_LRO (1 << 1)
  67
  68 /**
  69  * Initialize flow attributes structure according to flow items' types.
  70  *
  71  * @param[in] item
  72  *   Pointer to item specification.
  73  * @param[out] attr
  74  *   Pointer to flow attributes structure.
  75  */
  76 static void
  77 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  78 {
  79         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  80                 switch (item->type) {
  81                 case RTE_FLOW_ITEM_TYPE_IPV4:
  82                         attr->ipv4 = 1;
  83                         break;
  84                 case RTE_FLOW_ITEM_TYPE_IPV6:
  85                         attr->ipv6 = 1;
  86                         break;
  87                 case RTE_FLOW_ITEM_TYPE_UDP:
  88                         attr->udp = 1;
  89                         break;
  90                 case RTE_FLOW_ITEM_TYPE_TCP:
  91                         attr->tcp = 1;
  92                         break;
  93                 default:
  94                         break;
  95                 }
  96         }
  97         attr->valid = 1;
  98 }
  99
 100 struct field_modify_info {
 101         uint32_t size; /* Size of field in protocol header, in bytes. */
 102         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 103         enum mlx5_modification_field id;
 104 };
 105
 106 struct field_modify_info modify_eth[] = {
 107         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 108         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 109         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 110         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 111         {0, 0, 0},
 112 };
 113
 114 struct field_modify_info modify_ipv4[] = {
 115         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 116         {4, 12, MLX5_MODI_OUT_SIPV4},
 117         {4, 16, MLX5_MODI_OUT_DIPV4},
 118         {0, 0, 0},
 119 };
 120
 121 struct field_modify_info modify_ipv6[] = {
 122         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 123         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 124         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 125         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 126         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 127         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 128         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 129         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 130         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 131         {0, 0, 0},
 132 };
 133
 134 struct field_modify_info modify_udp[] = {
 135         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 136         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 137         {0, 0, 0},
 138 };
 139
 140 struct field_modify_info modify_tcp[] = {
 141         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 142         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 143         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 144         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 145         {0, 0, 0},
 146 };
 147
 148 static void
 149 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item, uint64_t *flags)
 150 {
 151         uint8_t next_protocol = 0xFF;
 152
 153         if (item->mask != NULL) {
 154                 switch (item->type) {
 155                 case RTE_FLOW_ITEM_TYPE_IPV4:
 156                         next_protocol =
 157                                 ((const struct rte_flow_item_ipv4 *)
 158                                  (item->spec))->hdr.next_proto_id;
 159                         next_protocol &=
 160                                 ((const struct rte_flow_item_ipv4 *)
 161                                  (item->mask))->hdr.next_proto_id;
 162                         break;
 163                 case RTE_FLOW_ITEM_TYPE_IPV6:
 164                         next_protocol =
 165                                 ((const struct rte_flow_item_ipv6 *)
 166                                  (item->spec))->hdr.proto;
 167                         next_protocol &=
 168                                 ((const struct rte_flow_item_ipv6 *)
 169                                  (item->mask))->hdr.proto;
 170                         break;
 171                 default:
 172                         break;
 173                 }
 174         }
 175         if (next_protocol == IPPROTO_IPIP)
 176                 *flags |= MLX5_FLOW_LAYER_IPIP;
 177         if (next_protocol == IPPROTO_IPV6)
 178                 *flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 179 }
 180
 181 /**
 182  * Acquire the synchronizing object to protect multithreaded access
 183  * to shared dv context. Lock occurs only if context is actually
 184  * shared, i.e. we have multiport IB device and representors are
 185  * created.
 186  *
 187  * @param[in] dev
 188  *   Pointer to the rte_eth_dev structure.
 189  */
 190 static void
 191 flow_d_shared_lock(struct rte_eth_dev *dev)
 192 {
 193         struct mlx5_priv *priv = dev->data->dev_private;
 194         struct mlx5_ibv_shared *sh = priv->sh;
 195
 196         if (sh->dv_refcnt > 1) {
 197                 int ret;
 198
 199                 ret = pthread_mutex_lock(&sh->dv_mutex);
 200                 assert(!ret);
 201                 (void)ret;
 202         }
 203 }
 204
 205 static void
 206 flow_d_shared_unlock(struct rte_eth_dev *dev)
 207 {
 208         struct mlx5_priv *priv = dev->data->dev_private;
 209         struct mlx5_ibv_shared *sh = priv->sh;
 210
 211         if (sh->dv_refcnt > 1) {
 212                 int ret;
 213
 214                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 215                 assert(!ret);
 216                 (void)ret;
 217         }
 218 }
 219
 220 /**
 221  * Convert modify-header action to DV specification.
 222  *
 223  * @param[in] item
 224  *   Pointer to item specification.
 225  * @param[in] field
 226  *   Pointer to field modification information.
 227  * @param[in,out] resource
 228  *   Pointer to the modify-header resource.
 229  * @param[in] type
 230  *   Type of modification.
 231  * @param[out] error
 232  *   Pointer to the error structure.
 233  *
 234  * @return
 235  *   0 on success, a negative errno value otherwise and rte_errno is set.
 236  */
 237 static int
 238 flow_dv_convert_modify_action(struct rte_flow_item *item,
 239                               struct field_modify_info *field,
 240                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 241                               uint32_t type,
 242                               struct rte_flow_error *error)
 243 {
 244         uint32_t i = resource->actions_num;
 245         struct mlx5_modification_cmd *actions = resource->actions;
 246         const uint8_t *spec = item->spec;
 247         const uint8_t *mask = item->mask;
 248         uint32_t set;
 249
 250         while (field->size) {
 251                 set = 0;
 252                 /* Generate modify command for each mask segment. */
 253                 memcpy(&set, &mask[field->offset], field->size);
 254                 if (set) {
 255                         if (i >= MLX5_MODIFY_NUM)
 256                                 return rte_flow_error_set(error, EINVAL,
 257                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 258                                          "too many items to modify");
 259                         actions[i].action_type = type;
 260                         actions[i].field = field->id;
 261                         actions[i].length = field->size ==
 262                                         4 ? 0 : field->size * 8;
 263                         rte_memcpy(&actions[i].data[4 - field->size],
 264                                    &spec[field->offset], field->size);
 265                         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 266                         ++i;
 267                 }
 268                 if (resource->actions_num != i)
 269                         resource->actions_num = i;
 270                 field++;
 271         }
 272         if (!resource->actions_num)
 273                 return rte_flow_error_set(error, EINVAL,
 274                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 275                                           "invalid modification flow item");
 276         return 0;
 277 }
 278
 279 /**
 280  * Convert modify-header set IPv4 address action to DV specification.
 281  *
 282  * @param[in,out] resource
 283  *   Pointer to the modify-header resource.
 284  * @param[in] action
 285  *   Pointer to action specification.
 286  * @param[out] error
 287  *   Pointer to the error structure.
 288  *
 289  * @return
 290  *   0 on success, a negative errno value otherwise and rte_errno is set.
 291  */
 292 static int
 293 flow_dv_convert_action_modify_ipv4
 294                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 295                          const struct rte_flow_action *action,
 296                          struct rte_flow_error *error)
 297 {
 298         const struct rte_flow_action_set_ipv4 *conf =
 299                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 300         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 301         struct rte_flow_item_ipv4 ipv4;
 302         struct rte_flow_item_ipv4 ipv4_mask;
 303
 304         memset(&ipv4, 0, sizeof(ipv4));
 305         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 306         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 307                 ipv4.hdr.src_addr = conf->ipv4_addr;
 308                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 309         } else {
 310                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 311                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 312         }
 313         item.spec = &ipv4;
 314         item.mask = &ipv4_mask;
 315         return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
 316                                              MLX5_MODIFICATION_TYPE_SET, error);
 317 }
 318
 319 /**
 320  * Convert modify-header set IPv6 address action to DV specification.
 321  *
 322  * @param[in,out] resource
 323  *   Pointer to the modify-header resource.
 324  * @param[in] action
 325  *   Pointer to action specification.
 326  * @param[out] error
 327  *   Pointer to the error structure.
 328  *
 329  * @return
 330  *   0 on success, a negative errno value otherwise and rte_errno is set.
 331  */
 332 static int
 333 flow_dv_convert_action_modify_ipv6
 334                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 335                          const struct rte_flow_action *action,
 336                          struct rte_flow_error *error)
 337 {
 338         const struct rte_flow_action_set_ipv6 *conf =
 339                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 340         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 341         struct rte_flow_item_ipv6 ipv6;
 342         struct rte_flow_item_ipv6 ipv6_mask;
 343
 344         memset(&ipv6, 0, sizeof(ipv6));
 345         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 346         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 347                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 348                        sizeof(ipv6.hdr.src_addr));
 349                 memcpy(&ipv6_mask.hdr.src_addr,
 350                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 351                        sizeof(ipv6.hdr.src_addr));
 352         } else {
 353                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 354                        sizeof(ipv6.hdr.dst_addr));
 355                 memcpy(&ipv6_mask.hdr.dst_addr,
 356                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 357                        sizeof(ipv6.hdr.dst_addr));
 358         }
 359         item.spec = &ipv6;
 360         item.mask = &ipv6_mask;
 361         return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
 362                                              MLX5_MODIFICATION_TYPE_SET, error);
 363 }
 364
 365 /**
 366  * Convert modify-header set MAC address action to DV specification.
 367  *
 368  * @param[in,out] resource
 369  *   Pointer to the modify-header resource.
 370  * @param[in] action
 371  *   Pointer to action specification.
 372  * @param[out] error
 373  *   Pointer to the error structure.
 374  *
 375  * @return
 376  *   0 on success, a negative errno value otherwise and rte_errno is set.
 377  */
 378 static int
 379 flow_dv_convert_action_modify_mac
 380                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 381                          const struct rte_flow_action *action,
 382                          struct rte_flow_error *error)
 383 {
 384         const struct rte_flow_action_set_mac *conf =
 385                 (const struct rte_flow_action_set_mac *)(action->conf);
 386         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 387         struct rte_flow_item_eth eth;
 388         struct rte_flow_item_eth eth_mask;
 389
 390         memset(&eth, 0, sizeof(eth));
 391         memset(&eth_mask, 0, sizeof(eth_mask));
 392         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 393                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 394                        sizeof(eth.src.addr_bytes));
 395                 memcpy(&eth_mask.src.addr_bytes,
 396                        &rte_flow_item_eth_mask.src.addr_bytes,
 397                        sizeof(eth_mask.src.addr_bytes));
 398         } else {
 399                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 400                        sizeof(eth.dst.addr_bytes));
 401                 memcpy(&eth_mask.dst.addr_bytes,
 402                        &rte_flow_item_eth_mask.dst.addr_bytes,
 403                        sizeof(eth_mask.dst.addr_bytes));
 404         }
 405         item.spec = &eth;
 406         item.mask = &eth_mask;
 407         return flow_dv_convert_modify_action(&item, modify_eth, resource,
 408                                              MLX5_MODIFICATION_TYPE_SET, error);
 409 }
 410
 411 /**
 412  * Convert modify-header set TP action to DV specification.
 413  *
 414  * @param[in,out] resource
 415  *   Pointer to the modify-header resource.
 416  * @param[in] action
 417  *   Pointer to action specification.
 418  * @param[in] items
 419  *   Pointer to rte_flow_item objects list.
 420  * @param[in] attr
 421  *   Pointer to flow attributes structure.
 422  * @param[out] error
 423  *   Pointer to the error structure.
 424  *
 425  * @return
 426  *   0 on success, a negative errno value otherwise and rte_errno is set.
 427  */
 428 static int
 429 flow_dv_convert_action_modify_tp
 430                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 431                          const struct rte_flow_action *action,
 432                          const struct rte_flow_item *items,
 433                          union flow_dv_attr *attr,
 434                          struct rte_flow_error *error)
 435 {
 436         const struct rte_flow_action_set_tp *conf =
 437                 (const struct rte_flow_action_set_tp *)(action->conf);
 438         struct rte_flow_item item;
 439         struct rte_flow_item_udp udp;
 440         struct rte_flow_item_udp udp_mask;
 441         struct rte_flow_item_tcp tcp;
 442         struct rte_flow_item_tcp tcp_mask;
 443         struct field_modify_info *field;
 444
 445         if (!attr->valid)
 446                 flow_dv_attr_init(items, attr);
 447         if (attr->udp) {
 448                 memset(&udp, 0, sizeof(udp));
 449                 memset(&udp_mask, 0, sizeof(udp_mask));
 450                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 451                         udp.hdr.src_port = conf->port;
 452                         udp_mask.hdr.src_port =
 453                                         rte_flow_item_udp_mask.hdr.src_port;
 454                 } else {
 455                         udp.hdr.dst_port = conf->port;
 456                         udp_mask.hdr.dst_port =
 457                                         rte_flow_item_udp_mask.hdr.dst_port;
 458                 }
 459                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 460                 item.spec = &udp;
 461                 item.mask = &udp_mask;
 462                 field = modify_udp;
 463         }
 464         if (attr->tcp) {
 465                 memset(&tcp, 0, sizeof(tcp));
 466                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 467                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 468                         tcp.hdr.src_port = conf->port;
 469                         tcp_mask.hdr.src_port =
 470                                         rte_flow_item_tcp_mask.hdr.src_port;
 471                 } else {
 472                         tcp.hdr.dst_port = conf->port;
 473                         tcp_mask.hdr.dst_port =
 474                                         rte_flow_item_tcp_mask.hdr.dst_port;
 475                 }
 476                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 477                 item.spec = &tcp;
 478                 item.mask = &tcp_mask;
 479                 field = modify_tcp;
 480         }
 481         return flow_dv_convert_modify_action(&item, field, resource,
 482                                              MLX5_MODIFICATION_TYPE_SET, error);
 483 }
 484
 485 /**
 486  * Convert modify-header set TTL action to DV specification.
 487  *
 488  * @param[in,out] resource
 489  *   Pointer to the modify-header resource.
 490  * @param[in] action
 491  *   Pointer to action specification.
 492  * @param[in] items
 493  *   Pointer to rte_flow_item objects list.
 494  * @param[in] attr
 495  *   Pointer to flow attributes structure.
 496  * @param[out] error
 497  *   Pointer to the error structure.
 498  *
 499  * @return
 500  *   0 on success, a negative errno value otherwise and rte_errno is set.
 501  */
 502 static int
 503 flow_dv_convert_action_modify_ttl
 504                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 505                          const struct rte_flow_action *action,
 506                          const struct rte_flow_item *items,
 507                          union flow_dv_attr *attr,
 508                          struct rte_flow_error *error)
 509 {
 510         const struct rte_flow_action_set_ttl *conf =
 511                 (const struct rte_flow_action_set_ttl *)(action->conf);
 512         struct rte_flow_item item;
 513         struct rte_flow_item_ipv4 ipv4;
 514         struct rte_flow_item_ipv4 ipv4_mask;
 515         struct rte_flow_item_ipv6 ipv6;
 516         struct rte_flow_item_ipv6 ipv6_mask;
 517         struct field_modify_info *field;
 518
 519         if (!attr->valid)
 520                 flow_dv_attr_init(items, attr);
 521         if (attr->ipv4) {
 522                 memset(&ipv4, 0, sizeof(ipv4));
 523                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 524                 ipv4.hdr.time_to_live = conf->ttl_value;
 525                 ipv4_mask.hdr.time_to_live = 0xFF;
 526                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 527                 item.spec = &ipv4;
 528                 item.mask = &ipv4_mask;
 529                 field = modify_ipv4;
 530         }
 531         if (attr->ipv6) {
 532                 memset(&ipv6, 0, sizeof(ipv6));
 533                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 534                 ipv6.hdr.hop_limits = conf->ttl_value;
 535                 ipv6_mask.hdr.hop_limits = 0xFF;
 536                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 537                 item.spec = &ipv6;
 538                 item.mask = &ipv6_mask;
 539                 field = modify_ipv6;
 540         }
 541         return flow_dv_convert_modify_action(&item, field, resource,
 542                                              MLX5_MODIFICATION_TYPE_SET, error);
 543 }
 544
 545 /**
 546  * Convert modify-header decrement TTL action to DV specification.
 547  *
 548  * @param[in,out] resource
 549  *   Pointer to the modify-header resource.
 550  * @param[in] action
 551  *   Pointer to action specification.
 552  * @param[in] items
 553  *   Pointer to rte_flow_item objects list.
 554  * @param[in] attr
 555  *   Pointer to flow attributes structure.
 556  * @param[out] error
 557  *   Pointer to the error structure.
 558  *
 559  * @return
 560  *   0 on success, a negative errno value otherwise and rte_errno is set.
 561  */
 562 static int
 563 flow_dv_convert_action_modify_dec_ttl
 564                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 565                          const struct rte_flow_item *items,
 566                          union flow_dv_attr *attr,
 567                          struct rte_flow_error *error)
 568 {
 569         struct rte_flow_item item;
 570         struct rte_flow_item_ipv4 ipv4;
 571         struct rte_flow_item_ipv4 ipv4_mask;
 572         struct rte_flow_item_ipv6 ipv6;
 573         struct rte_flow_item_ipv6 ipv6_mask;
 574         struct field_modify_info *field;
 575
 576         if (!attr->valid)
 577                 flow_dv_attr_init(items, attr);
 578         if (attr->ipv4) {
 579                 memset(&ipv4, 0, sizeof(ipv4));
 580                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 581                 ipv4.hdr.time_to_live = 0xFF;
 582                 ipv4_mask.hdr.time_to_live = 0xFF;
 583                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 584                 item.spec = &ipv4;
 585                 item.mask = &ipv4_mask;
 586                 field = modify_ipv4;
 587         }
 588         if (attr->ipv6) {
 589                 memset(&ipv6, 0, sizeof(ipv6));
 590                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 591                 ipv6.hdr.hop_limits = 0xFF;
 592                 ipv6_mask.hdr.hop_limits = 0xFF;
 593                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 594                 item.spec = &ipv6;
 595                 item.mask = &ipv6_mask;
 596                 field = modify_ipv6;
 597         }
 598         return flow_dv_convert_modify_action(&item, field, resource,
 599                                              MLX5_MODIFICATION_TYPE_ADD, error);
 600 }
 601
 602 /**
 603  * Convert modify-header increment/decrement TCP Sequence number
 604  * to DV specification.
 605  *
 606  * @param[in,out] resource
 607  *   Pointer to the modify-header resource.
 608  * @param[in] action
 609  *   Pointer to action specification.
 610  * @param[out] error
 611  *   Pointer to the error structure.
 612  *
 613  * @return
 614  *   0 on success, a negative errno value otherwise and rte_errno is set.
 615  */
 616 static int
 617 flow_dv_convert_action_modify_tcp_seq
 618                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 619                          const struct rte_flow_action *action,
 620                          struct rte_flow_error *error)
 621 {
 622         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 623         uint64_t value = rte_be_to_cpu_32(*conf);
 624         struct rte_flow_item item;
 625         struct rte_flow_item_tcp tcp;
 626         struct rte_flow_item_tcp tcp_mask;
 627
 628         memset(&tcp, 0, sizeof(tcp));
 629         memset(&tcp_mask, 0, sizeof(tcp_mask));
 630         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 631                 /*
 632                  * The HW has no decrement operation, only increment operation.
 633                  * To simulate decrement X from Y using increment operation
 634                  * we need to add UINT32_MAX X times to Y.
 635                  * Each adding of UINT32_MAX decrements Y by 1.
 636                  */
 637                 value *= UINT32_MAX;
 638         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 639         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 640         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 641         item.spec = &tcp;
 642         item.mask = &tcp_mask;
 643         return flow_dv_convert_modify_action(&item, modify_tcp, resource,
 644                                              MLX5_MODIFICATION_TYPE_ADD, error);
 645 }
 646
 647 /**
 648  * Convert modify-header increment/decrement TCP Acknowledgment number
 649  * to DV specification.
 650  *
 651  * @param[in,out] resource
 652  *   Pointer to the modify-header resource.
 653  * @param[in] action
 654  *   Pointer to action specification.
 655  * @param[out] error
 656  *   Pointer to the error structure.
 657  *
 658  * @return
 659  *   0 on success, a negative errno value otherwise and rte_errno is set.
 660  */
 661 static int
 662 flow_dv_convert_action_modify_tcp_ack
 663                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 664                          const struct rte_flow_action *action,
 665                          struct rte_flow_error *error)
 666 {
 667         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 668         uint64_t value = rte_be_to_cpu_32(*conf);
 669         struct rte_flow_item item;
 670         struct rte_flow_item_tcp tcp;
 671         struct rte_flow_item_tcp tcp_mask;
 672
 673         memset(&tcp, 0, sizeof(tcp));
 674         memset(&tcp_mask, 0, sizeof(tcp_mask));
 675         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 676                 /*
 677                  * The HW has no decrement operation, only increment operation.
 678                  * To simulate decrement X from Y using increment operation
 679                  * we need to add UINT32_MAX X times to Y.
 680                  * Each adding of UINT32_MAX decrements Y by 1.
 681                  */
 682                 value *= UINT32_MAX;
 683         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 684         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 685         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 686         item.spec = &tcp;
 687         item.mask = &tcp_mask;
 688         return flow_dv_convert_modify_action(&item, modify_tcp, resource,
 689                                              MLX5_MODIFICATION_TYPE_ADD, error);
 690 }
 691
 692 /**
 693  * Validate META item.
 694  *
 695  * @param[in] dev
 696  *   Pointer to the rte_eth_dev structure.
 697  * @param[in] item
 698  *   Item specification.
 699  * @param[in] attr
 700  *   Attributes of flow that includes this item.
 701  * @param[out] error
 702  *   Pointer to error structure.
 703  *
 704  * @return
 705  *   0 on success, a negative errno value otherwise and rte_errno is set.
 706  */
 707 static int
 708 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
 709                            const struct rte_flow_item *item,
 710                            const struct rte_flow_attr *attr,
 711                            struct rte_flow_error *error)
 712 {
 713         const struct rte_flow_item_meta *spec = item->spec;
 714         const struct rte_flow_item_meta *mask = item->mask;
 715         const struct rte_flow_item_meta nic_mask = {
 716                 .data = RTE_BE32(UINT32_MAX)
 717         };
 718         int ret;
 719         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
 720
 721         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
 722                 return rte_flow_error_set(error, EPERM,
 723                                           RTE_FLOW_ERROR_TYPE_ITEM,
 724                                           NULL,
 725                                           "match on metadata offload "
 726                                           "configuration is off for this port");
 727         if (!spec)
 728                 return rte_flow_error_set(error, EINVAL,
 729                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 730                                           item->spec,
 731                                           "data cannot be empty");
 732         if (!spec->data)
 733                 return rte_flow_error_set(error, EINVAL,
 734                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 735                                           NULL,
 736                                           "data cannot be zero");
 737         if (!mask)
 738                 mask = &rte_flow_item_meta_mask;
 739         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 740                                         (const uint8_t *)&nic_mask,
 741                                         sizeof(struct rte_flow_item_meta),
 742                                         error);
 743         if (ret < 0)
 744                 return ret;
 745         if (attr->ingress)
 746                 return rte_flow_error_set(error, ENOTSUP,
 747                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 748                                           NULL,
 749                                           "pattern not supported for ingress");
 750         return 0;
 751 }
 752
 753 /**
 754  * Validate vport item.
 755  *
 756  * @param[in] dev
 757  *   Pointer to the rte_eth_dev structure.
 758  * @param[in] item
 759  *   Item specification.
 760  * @param[in] attr
 761  *   Attributes of flow that includes this item.
 762  * @param[in] item_flags
 763  *   Bit-fields that holds the items detected until now.
 764  * @param[out] error
 765  *   Pointer to error structure.
 766  *
 767  * @return
 768  *   0 on success, a negative errno value otherwise and rte_errno is set.
 769  */
 770 static int
 771 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
 772                               const struct rte_flow_item *item,
 773                               const struct rte_flow_attr *attr,
 774                               uint64_t item_flags,
 775                               struct rte_flow_error *error)
 776 {
 777         const struct rte_flow_item_port_id *spec = item->spec;
 778         const struct rte_flow_item_port_id *mask = item->mask;
 779         const struct rte_flow_item_port_id switch_mask = {
 780                         .id = 0xffffffff,
 781         };
 782         uint16_t esw_domain_id;
 783         uint16_t item_port_esw_domain_id;
 784         int ret;
 785
 786         if (!attr->transfer)
 787                 return rte_flow_error_set(error, EINVAL,
 788                                           RTE_FLOW_ERROR_TYPE_ITEM,
 789                                           NULL,
 790                                           "match on port id is valid only"
 791                                           " when transfer flag is enabled");
 792         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
 793                 return rte_flow_error_set(error, ENOTSUP,
 794                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 795                                           "multiple source ports are not"
 796                                           " supported");
 797         if (!mask)
 798                 mask = &switch_mask;
 799         if (mask->id != 0xffffffff)
 800                 return rte_flow_error_set(error, ENOTSUP,
 801                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
 802                                            mask,
 803                                            "no support for partial mask on"
 804                                            " \"id\" field");
 805         ret = mlx5_flow_item_acceptable
 806                                 (item, (const uint8_t *)mask,
 807                                  (const uint8_t *)&rte_flow_item_port_id_mask,
 808                                  sizeof(struct rte_flow_item_port_id),
 809                                  error);
 810         if (ret)
 811                 return ret;
 812         if (!spec)
 813                 return 0;
 814         ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
 815                                         NULL);
 816         if (ret)
 817                 return rte_flow_error_set(error, -ret,
 818                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
 819                                           "failed to obtain E-Switch info for"
 820                                           " port");
 821         ret = mlx5_port_to_eswitch_info(dev->data->port_id,
 822                                         &esw_domain_id, NULL);
 823         if (ret < 0)
 824                 return rte_flow_error_set(error, -ret,
 825                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 826                                           NULL,
 827                                           "failed to obtain E-Switch info");
 828         if (item_port_esw_domain_id != esw_domain_id)
 829                 return rte_flow_error_set(error, -ret,
 830                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
 831                                           "cannot match on a port from a"
 832                                           " different E-Switch");
 833         return 0;
 834 }
 835
 836 /**
 837  * Validate count action.
 838  *
 839  * @param[in] dev
 840  *   device otr.
 841  * @param[out] error
 842  *   Pointer to error structure.
 843  *
 844  * @return
 845  *   0 on success, a negative errno value otherwise and rte_errno is set.
 846  */
 847 static int
 848 flow_dv_validate_action_count(struct rte_eth_dev *dev,
 849                               struct rte_flow_error *error)
 850 {
 851         struct mlx5_priv *priv = dev->data->dev_private;
 852
 853         if (!priv->config.devx)
 854                 goto notsup_err;
 855 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
 856         return 0;
 857 #endif
 858 notsup_err:
 859         return rte_flow_error_set
 860                       (error, ENOTSUP,
 861                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 862                        NULL,
 863                        "count action not supported");
 864 }
 865
 866 /**
 867  * Validate the L2 encap action.
 868  *
 869  * @param[in] action_flags
 870  *   Holds the actions detected until now.
 871  * @param[in] action
 872  *   Pointer to the encap action.
 873  * @param[in] attr
 874  *   Pointer to flow attributes
 875  * @param[out] error
 876  *   Pointer to error structure.
 877  *
 878  * @return
 879  *   0 on success, a negative errno value otherwise and rte_errno is set.
 880  */
 881 static int
 882 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 883                                  const struct rte_flow_action *action,
 884                                  const struct rte_flow_attr *attr,
 885                                  struct rte_flow_error *error)
 886 {
 887         if (!(action->conf))
 888                 return rte_flow_error_set(error, EINVAL,
 889                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 890                                           "configuration cannot be null");
 891         if (action_flags & MLX5_FLOW_ACTION_DROP)
 892                 return rte_flow_error_set(error, EINVAL,
 893                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 894                                           "can't drop and encap in same flow");
 895         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 896                 return rte_flow_error_set(error, EINVAL,
 897                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 898                                           "can only have a single encap or"
 899                                           " decap action in a flow");
 900         if (!attr->transfer && attr->ingress)
 901                 return rte_flow_error_set(error, ENOTSUP,
 902                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 903                                           NULL,
 904                                           "encap action not supported for "
 905                                           "ingress");
 906         return 0;
 907 }
 908
 909 /**
 910  * Validate the L2 decap action.
 911  *
 912  * @param[in] action_flags
 913  *   Holds the actions detected until now.
 914  * @param[in] attr
 915  *   Pointer to flow attributes
 916  * @param[out] error
 917  *   Pointer to error structure.
 918  *
 919  * @return
 920  *   0 on success, a negative errno value otherwise and rte_errno is set.
 921  */
 922 static int
 923 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 924                                  const struct rte_flow_attr *attr,
 925                                  struct rte_flow_error *error)
 926 {
 927         if (action_flags & MLX5_FLOW_ACTION_DROP)
 928                 return rte_flow_error_set(error, EINVAL,
 929                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 930                                           "can't drop and decap in same flow");
 931         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 932                 return rte_flow_error_set(error, EINVAL,
 933                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 934                                           "can only have a single encap or"
 935                                           " decap action in a flow");
 936         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 937                 return rte_flow_error_set(error, EINVAL,
 938                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 939                                           "can't have decap action after"
 940                                           " modify action");
 941         if (attr->egress)
 942                 return rte_flow_error_set(error, ENOTSUP,
 943                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 944                                           NULL,
 945                                           "decap action not supported for "
 946                                           "egress");
 947         return 0;
 948 }
 949
 950 /**
 951  * Validate the raw encap action.
 952  *
 953  * @param[in] action_flags
 954  *   Holds the actions detected until now.
 955  * @param[in] action
 956  *   Pointer to the encap action.
 957  * @param[in] attr
 958  *   Pointer to flow attributes
 959  * @param[out] error
 960  *   Pointer to error structure.
 961  *
 962  * @return
 963  *   0 on success, a negative errno value otherwise and rte_errno is set.
 964  */
 965 static int
 966 flow_dv_validate_action_raw_encap(uint64_t action_flags,
 967                                   const struct rte_flow_action *action,
 968                                   const struct rte_flow_attr *attr,
 969                                   struct rte_flow_error *error)
 970 {
 971         if (!(action->conf))
 972                 return rte_flow_error_set(error, EINVAL,
 973                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 974                                           "configuration cannot be null");
 975         if (action_flags & MLX5_FLOW_ACTION_DROP)
 976                 return rte_flow_error_set(error, EINVAL,
 977                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 978                                           "can't drop and encap in same flow");
 979         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 980                 return rte_flow_error_set(error, EINVAL,
 981                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 982                                           "can only have a single encap"
 983                                           " action in a flow");
 984         /* encap without preceding decap is not supported for ingress */
 985         if (!attr->transfer &&  attr->ingress &&
 986             !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
 987                 return rte_flow_error_set(error, ENOTSUP,
 988                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 989                                           NULL,
 990                                           "encap action not supported for "
 991                                           "ingress");
 992         return 0;
 993 }
 994
 995 /**
 996  * Validate the raw decap action.
 997  *
 998  * @param[in] action_flags
 999  *   Holds the actions detected until now.
1000  * @param[in] action
1001  *   Pointer to the encap action.
1002  * @param[in] attr
1003  *   Pointer to flow attributes
1004  * @param[out] error
1005  *   Pointer to error structure.
1006  *
1007  * @return
1008  *   0 on success, a negative errno value otherwise and rte_errno is set.
1009  */
1010 static int
1011 flow_dv_validate_action_raw_decap(uint64_t action_flags,
1012                                   const struct rte_flow_action *action,
1013                                   const struct rte_flow_attr *attr,
1014                                   struct rte_flow_error *error)
1015 {
1016         if (action_flags & MLX5_FLOW_ACTION_DROP)
1017                 return rte_flow_error_set(error, EINVAL,
1018                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1019                                           "can't drop and decap in same flow");
1020         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1021                 return rte_flow_error_set(error, EINVAL,
1022                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1023                                           "can't have encap action before"
1024                                           " decap action");
1025         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1026                 return rte_flow_error_set(error, EINVAL,
1027                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1028                                           "can only have a single decap"
1029                                           " action in a flow");
1030         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1031                 return rte_flow_error_set(error, EINVAL,
1032                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1033                                           "can't have decap action after"
1034                                           " modify action");
1035         /* decap action is valid on egress only if it is followed by encap */
1036         if (attr->egress) {
1037                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1038                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1039                        action++) {
1040                 }
1041                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1042                         return rte_flow_error_set
1043                                         (error, ENOTSUP,
1044                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1045                                          NULL, "decap action not supported"
1046                                          " for egress");
1047         }
1048         return 0;
1049 }
1050
1051 /**
1052  * Find existing encap/decap resource or create and register a new one.
1053  *
1054  * @param dev[in, out]
1055  *   Pointer to rte_eth_dev structure.
1056  * @param[in, out] resource
1057  *   Pointer to encap/decap resource.
1058  * @parm[in, out] dev_flow
1059  *   Pointer to the dev_flow.
1060  * @param[out] error
1061  *   pointer to error structure.
1062  *
1063  * @return
1064  *   0 on success otherwise -errno and errno is set.
1065  */
1066 static int
1067 flow_dv_encap_decap_resource_register
1068                         (struct rte_eth_dev *dev,
1069                          struct mlx5_flow_dv_encap_decap_resource *resource,
1070                          struct mlx5_flow *dev_flow,
1071                          struct rte_flow_error *error)
1072 {
1073         struct mlx5_priv *priv = dev->data->dev_private;
1074         struct mlx5_ibv_shared *sh = priv->sh;
1075         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1076         struct rte_flow *flow = dev_flow->flow;
1077         struct mlx5dv_dr_domain *domain;
1078
1079         resource->flags = flow->group ? 0 : 1;
1080         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1081                 domain = sh->fdb_domain;
1082         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1083                 domain = sh->rx_domain;
1084         else
1085                 domain = sh->tx_domain;
1086
1087         /* Lookup a matching resource from cache. */
1088         LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1089                 if (resource->reformat_type == cache_resource->reformat_type &&
1090                     resource->ft_type == cache_resource->ft_type &&
1091                     resource->flags == cache_resource->flags &&
1092                     resource->size == cache_resource->size &&
1093                     !memcmp((const void *)resource->buf,
1094                             (const void *)cache_resource->buf,
1095                             resource->size)) {
1096                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1097                                 (void *)cache_resource,
1098                                 rte_atomic32_read(&cache_resource->refcnt));
1099                         rte_atomic32_inc(&cache_resource->refcnt);
1100                         dev_flow->dv.encap_decap = cache_resource;
1101                         return 0;
1102                 }
1103         }
1104         /* Register new encap/decap resource. */
1105         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1106         if (!cache_resource)
1107                 return rte_flow_error_set(error, ENOMEM,
1108                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1109                                           "cannot allocate resource memory");
1110         *cache_resource = *resource;
1111         cache_resource->verbs_action =
1112                 mlx5_glue->dv_create_flow_action_packet_reformat
1113                         (sh->ctx, cache_resource->reformat_type,
1114                          cache_resource->ft_type, domain, cache_resource->flags,
1115                          cache_resource->size,
1116                          (cache_resource->size ? cache_resource->buf : NULL));
1117         if (!cache_resource->verbs_action) {
1118                 rte_free(cache_resource);
1119                 return rte_flow_error_set(error, ENOMEM,
1120                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1121                                           NULL, "cannot create action");
1122         }
1123         rte_atomic32_init(&cache_resource->refcnt);
1124         rte_atomic32_inc(&cache_resource->refcnt);
1125         LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1126         dev_flow->dv.encap_decap = cache_resource;
1127         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1128                 (void *)cache_resource,
1129                 rte_atomic32_read(&cache_resource->refcnt));
1130         return 0;
1131 }
1132
1133 /**
1134  * Find existing table jump resource or create and register a new one.
1135  *
1136  * @param dev[in, out]
1137  *   Pointer to rte_eth_dev structure.
1138  * @param[in, out] resource
1139  *   Pointer to jump table resource.
1140  * @parm[in, out] dev_flow
1141  *   Pointer to the dev_flow.
1142  * @param[out] error
1143  *   pointer to error structure.
1144  *
1145  * @return
1146  *   0 on success otherwise -errno and errno is set.
1147  */
1148 static int
1149 flow_dv_jump_tbl_resource_register
1150                         (struct rte_eth_dev *dev,
1151                          struct mlx5_flow_dv_jump_tbl_resource *resource,
1152                          struct mlx5_flow *dev_flow,
1153                          struct rte_flow_error *error)
1154 {
1155         struct mlx5_priv *priv = dev->data->dev_private;
1156         struct mlx5_ibv_shared *sh = priv->sh;
1157         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1158
1159         /* Lookup a matching resource from cache. */
1160         LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1161                 if (resource->tbl == cache_resource->tbl) {
1162                         DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1163                                 (void *)cache_resource,
1164                                 rte_atomic32_read(&cache_resource->refcnt));
1165                         rte_atomic32_inc(&cache_resource->refcnt);
1166                         dev_flow->dv.jump = cache_resource;
1167                         return 0;
1168                 }
1169         }
1170         /* Register new jump table resource. */
1171         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1172         if (!cache_resource)
1173                 return rte_flow_error_set(error, ENOMEM,
1174                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1175                                           "cannot allocate resource memory");
1176         *cache_resource = *resource;
1177         cache_resource->action =
1178                 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1179                 (resource->tbl->obj);
1180         if (!cache_resource->action) {
1181                 rte_free(cache_resource);
1182                 return rte_flow_error_set(error, ENOMEM,
1183                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1184                                           NULL, "cannot create action");
1185         }
1186         rte_atomic32_init(&cache_resource->refcnt);
1187         rte_atomic32_inc(&cache_resource->refcnt);
1188         LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1189         dev_flow->dv.jump = cache_resource;
1190         DRV_LOG(DEBUG, "new jump table  resource %p: refcnt %d++",
1191                 (void *)cache_resource,
1192                 rte_atomic32_read(&cache_resource->refcnt));
1193         return 0;
1194 }
1195
1196 /**
1197  * Find existing table port ID resource or create and register a new one.
1198  *
1199  * @param dev[in, out]
1200  *   Pointer to rte_eth_dev structure.
1201  * @param[in, out] resource
1202  *   Pointer to port ID action resource.
1203  * @parm[in, out] dev_flow
1204  *   Pointer to the dev_flow.
1205  * @param[out] error
1206  *   pointer to error structure.
1207  *
1208  * @return
1209  *   0 on success otherwise -errno and errno is set.
1210  */
1211 static int
1212 flow_dv_port_id_action_resource_register
1213                         (struct rte_eth_dev *dev,
1214                          struct mlx5_flow_dv_port_id_action_resource *resource,
1215                          struct mlx5_flow *dev_flow,
1216                          struct rte_flow_error *error)
1217 {
1218         struct mlx5_priv *priv = dev->data->dev_private;
1219         struct mlx5_ibv_shared *sh = priv->sh;
1220         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1221
1222         /* Lookup a matching resource from cache. */
1223         LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1224                 if (resource->port_id == cache_resource->port_id) {
1225                         DRV_LOG(DEBUG, "port id action resource resource %p: "
1226                                 "refcnt %d++",
1227                                 (void *)cache_resource,
1228                                 rte_atomic32_read(&cache_resource->refcnt));
1229                         rte_atomic32_inc(&cache_resource->refcnt);
1230                         dev_flow->dv.port_id_action = cache_resource;
1231                         return 0;
1232                 }
1233         }
1234         /* Register new port id action resource. */
1235         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1236         if (!cache_resource)
1237                 return rte_flow_error_set(error, ENOMEM,
1238                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1239                                           "cannot allocate resource memory");
1240         *cache_resource = *resource;
1241         cache_resource->action =
1242                 mlx5_glue->dr_create_flow_action_dest_vport
1243                         (priv->sh->fdb_domain, resource->port_id);
1244         if (!cache_resource->action) {
1245                 rte_free(cache_resource);
1246                 return rte_flow_error_set(error, ENOMEM,
1247                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1248                                           NULL, "cannot create action");
1249         }
1250         rte_atomic32_init(&cache_resource->refcnt);
1251         rte_atomic32_inc(&cache_resource->refcnt);
1252         LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1253         dev_flow->dv.port_id_action = cache_resource;
1254         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1255                 (void *)cache_resource,
1256                 rte_atomic32_read(&cache_resource->refcnt));
1257         return 0;
1258 }
1259
1260 /**
1261  * Get the size of specific rte_flow_item_type
1262  *
1263  * @param[in] item_type
1264  *   Tested rte_flow_item_type.
1265  *
1266  * @return
1267  *   sizeof struct item_type, 0 if void or irrelevant.
1268  */
1269 static size_t
1270 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1271 {
1272         size_t retval;
1273
1274         switch (item_type) {
1275         case RTE_FLOW_ITEM_TYPE_ETH:
1276                 retval = sizeof(struct rte_flow_item_eth);
1277                 break;
1278         case RTE_FLOW_ITEM_TYPE_VLAN:
1279                 retval = sizeof(struct rte_flow_item_vlan);
1280                 break;
1281         case RTE_FLOW_ITEM_TYPE_IPV4:
1282                 retval = sizeof(struct rte_flow_item_ipv4);
1283                 break;
1284         case RTE_FLOW_ITEM_TYPE_IPV6:
1285                 retval = sizeof(struct rte_flow_item_ipv6);
1286                 break;
1287         case RTE_FLOW_ITEM_TYPE_UDP:
1288                 retval = sizeof(struct rte_flow_item_udp);
1289                 break;
1290         case RTE_FLOW_ITEM_TYPE_TCP:
1291                 retval = sizeof(struct rte_flow_item_tcp);
1292                 break;
1293         case RTE_FLOW_ITEM_TYPE_VXLAN:
1294                 retval = sizeof(struct rte_flow_item_vxlan);
1295                 break;
1296         case RTE_FLOW_ITEM_TYPE_GRE:
1297                 retval = sizeof(struct rte_flow_item_gre);
1298                 break;
1299         case RTE_FLOW_ITEM_TYPE_NVGRE:
1300                 retval = sizeof(struct rte_flow_item_nvgre);
1301                 break;
1302         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1303                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1304                 break;
1305         case RTE_FLOW_ITEM_TYPE_MPLS:
1306                 retval = sizeof(struct rte_flow_item_mpls);
1307                 break;
1308         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1309         default:
1310                 retval = 0;
1311                 break;
1312         }
1313         return retval;
1314 }
1315
1316 #define MLX5_ENCAP_IPV4_VERSION         0x40
1317 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
1318 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
1319 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
1320 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
1321 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
1322 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
1323
1324 /**
1325  * Convert the encap action data from list of rte_flow_item to raw buffer
1326  *
1327  * @param[in] items
1328  *   Pointer to rte_flow_item objects list.
1329  * @param[out] buf
1330  *   Pointer to the output buffer.
1331  * @param[out] size
1332  *   Pointer to the output buffer size.
1333  * @param[out] error
1334  *   Pointer to the error structure.
1335  *
1336  * @return
1337  *   0 on success, a negative errno value otherwise and rte_errno is set.
1338  */
1339 static int
1340 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1341                            size_t *size, struct rte_flow_error *error)
1342 {
1343         struct rte_ether_hdr *eth = NULL;
1344         struct rte_vlan_hdr *vlan = NULL;
1345         struct rte_ipv4_hdr *ipv4 = NULL;
1346         struct rte_ipv6_hdr *ipv6 = NULL;
1347         struct rte_udp_hdr *udp = NULL;
1348         struct rte_vxlan_hdr *vxlan = NULL;
1349         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1350         struct rte_gre_hdr *gre = NULL;
1351         size_t len;
1352         size_t temp_size = 0;
1353
1354         if (!items)
1355                 return rte_flow_error_set(error, EINVAL,
1356                                           RTE_FLOW_ERROR_TYPE_ACTION,
1357                                           NULL, "invalid empty data");
1358         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1359                 len = flow_dv_get_item_len(items->type);
1360                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1361                         return rte_flow_error_set(error, EINVAL,
1362                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1363                                                   (void *)items->type,
1364                                                   "items total size is too big"
1365                                                   " for encap action");
1366                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1367                 switch (items->type) {
1368                 case RTE_FLOW_ITEM_TYPE_ETH:
1369                         eth = (struct rte_ether_hdr *)&buf[temp_size];
1370                         break;
1371                 case RTE_FLOW_ITEM_TYPE_VLAN:
1372                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1373                         if (!eth)
1374                                 return rte_flow_error_set(error, EINVAL,
1375                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1376                                                 (void *)items->type,
1377                                                 "eth header not found");
1378                         if (!eth->ether_type)
1379                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1380                         break;
1381                 case RTE_FLOW_ITEM_TYPE_IPV4:
1382                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1383                         if (!vlan && !eth)
1384                                 return rte_flow_error_set(error, EINVAL,
1385                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1386                                                 (void *)items->type,
1387                                                 "neither eth nor vlan"
1388                                                 " header found");
1389                         if (vlan && !vlan->eth_proto)
1390                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1391                         else if (eth && !eth->ether_type)
1392                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1393                         if (!ipv4->version_ihl)
1394                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1395                                                     MLX5_ENCAP_IPV4_IHL_MIN;
1396                         if (!ipv4->time_to_live)
1397                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1398                         break;
1399                 case RTE_FLOW_ITEM_TYPE_IPV6:
1400                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1401                         if (!vlan && !eth)
1402                                 return rte_flow_error_set(error, EINVAL,
1403                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1404                                                 (void *)items->type,
1405                                                 "neither eth nor vlan"
1406                                                 " header found");
1407                         if (vlan && !vlan->eth_proto)
1408                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1409                         else if (eth && !eth->ether_type)
1410                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1411                         if (!ipv6->vtc_flow)
1412                                 ipv6->vtc_flow =
1413                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1414                         if (!ipv6->hop_limits)
1415                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1416                         break;
1417                 case RTE_FLOW_ITEM_TYPE_UDP:
1418                         udp = (struct rte_udp_hdr *)&buf[temp_size];
1419                         if (!ipv4 && !ipv6)
1420                                 return rte_flow_error_set(error, EINVAL,
1421                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1422                                                 (void *)items->type,
1423                                                 "ip header not found");
1424                         if (ipv4 && !ipv4->next_proto_id)
1425                                 ipv4->next_proto_id = IPPROTO_UDP;
1426                         else if (ipv6 && !ipv6->proto)
1427                                 ipv6->proto = IPPROTO_UDP;
1428                         break;
1429                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1430                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1431                         if (!udp)
1432                                 return rte_flow_error_set(error, EINVAL,
1433                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1434                                                 (void *)items->type,
1435                                                 "udp header not found");
1436                         if (!udp->dst_port)
1437                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1438                         if (!vxlan->vx_flags)
1439                                 vxlan->vx_flags =
1440                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1441                         break;
1442                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1443                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1444                         if (!udp)
1445                                 return rte_flow_error_set(error, EINVAL,
1446                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1447                                                 (void *)items->type,
1448                                                 "udp header not found");
1449                         if (!vxlan_gpe->proto)
1450                                 return rte_flow_error_set(error, EINVAL,
1451                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1452                                                 (void *)items->type,
1453                                                 "next protocol not found");
1454                         if (!udp->dst_port)
1455                                 udp->dst_port =
1456                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1457                         if (!vxlan_gpe->vx_flags)
1458                                 vxlan_gpe->vx_flags =
1459                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1460                         break;
1461                 case RTE_FLOW_ITEM_TYPE_GRE:
1462                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1463                         gre = (struct rte_gre_hdr *)&buf[temp_size];
1464                         if (!gre->proto)
1465                                 return rte_flow_error_set(error, EINVAL,
1466                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1467                                                 (void *)items->type,
1468                                                 "next protocol not found");
1469                         if (!ipv4 && !ipv6)
1470                                 return rte_flow_error_set(error, EINVAL,
1471                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1472                                                 (void *)items->type,
1473                                                 "ip header not found");
1474                         if (ipv4 && !ipv4->next_proto_id)
1475                                 ipv4->next_proto_id = IPPROTO_GRE;
1476                         else if (ipv6 && !ipv6->proto)
1477                                 ipv6->proto = IPPROTO_GRE;
1478                         break;
1479                 case RTE_FLOW_ITEM_TYPE_VOID:
1480                         break;
1481                 default:
1482                         return rte_flow_error_set(error, EINVAL,
1483                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1484                                                   (void *)items->type,
1485                                                   "unsupported item type");
1486                         break;
1487                 }
1488                 temp_size += len;
1489         }
1490         *size = temp_size;
1491         return 0;
1492 }
1493
1494 static int
1495 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1496 {
1497         struct rte_ether_hdr *eth = NULL;
1498         struct rte_vlan_hdr *vlan = NULL;
1499         struct rte_ipv6_hdr *ipv6 = NULL;
1500         struct rte_udp_hdr *udp = NULL;
1501         char *next_hdr;
1502         uint16_t proto;
1503
1504         eth = (struct rte_ether_hdr *)data;
1505         next_hdr = (char *)(eth + 1);
1506         proto = RTE_BE16(eth->ether_type);
1507
1508         /* VLAN skipping */
1509         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1510                 next_hdr += sizeof(struct rte_vlan_hdr);
1511                 vlan = (struct rte_vlan_hdr *)next_hdr;
1512                 proto = RTE_BE16(vlan->eth_proto);
1513         }
1514
1515         /* HW calculates IPv4 csum. no need to proceed */
1516         if (proto == RTE_ETHER_TYPE_IPV4)
1517                 return 0;
1518
1519         /* non IPv4/IPv6 header. not supported */
1520         if (proto != RTE_ETHER_TYPE_IPV6) {
1521                 return rte_flow_error_set(error, ENOTSUP,
1522                                           RTE_FLOW_ERROR_TYPE_ACTION,
1523                                           NULL, "Cannot offload non IPv4/IPv6");
1524         }
1525
1526         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1527
1528         /* ignore non UDP */
1529         if (ipv6->proto != IPPROTO_UDP)
1530                 return 0;
1531
1532         udp = (struct rte_udp_hdr *)(ipv6 + 1);
1533         udp->dgram_cksum = 0;
1534
1535         return 0;
1536 }
1537
1538 /**
1539  * Convert L2 encap action to DV specification.
1540  *
1541  * @param[in] dev
1542  *   Pointer to rte_eth_dev structure.
1543  * @param[in] action
1544  *   Pointer to action structure.
1545  * @param[in, out] dev_flow
1546  *   Pointer to the mlx5_flow.
1547  * @param[in] transfer
1548  *   Mark if the flow is E-Switch flow.
1549  * @param[out] error
1550  *   Pointer to the error structure.
1551  *
1552  * @return
1553  *   0 on success, a negative errno value otherwise and rte_errno is set.
1554  */
1555 static int
1556 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1557                                const struct rte_flow_action *action,
1558                                struct mlx5_flow *dev_flow,
1559                                uint8_t transfer,
1560                                struct rte_flow_error *error)
1561 {
1562         const struct rte_flow_item *encap_data;
1563         const struct rte_flow_action_raw_encap *raw_encap_data;
1564         struct mlx5_flow_dv_encap_decap_resource res = {
1565                 .reformat_type =
1566                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1567                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1568                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1569         };
1570
1571         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1572                 raw_encap_data =
1573                         (const struct rte_flow_action_raw_encap *)action->conf;
1574                 res.size = raw_encap_data->size;
1575                 memcpy(res.buf, raw_encap_data->data, res.size);
1576                 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1577                         return -rte_errno;
1578         } else {
1579                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1580                         encap_data =
1581                                 ((const struct rte_flow_action_vxlan_encap *)
1582                                                 action->conf)->definition;
1583                 else
1584                         encap_data =
1585                                 ((const struct rte_flow_action_nvgre_encap *)
1586                                                 action->conf)->definition;
1587                 if (flow_dv_convert_encap_data(encap_data, res.buf,
1588                                                &res.size, error))
1589                         return -rte_errno;
1590         }
1591         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1592                 return rte_flow_error_set(error, EINVAL,
1593                                           RTE_FLOW_ERROR_TYPE_ACTION,
1594                                           NULL, "can't create L2 encap action");
1595         return 0;
1596 }
1597
1598 /**
1599  * Convert L2 decap action to DV specification.
1600  *
1601  * @param[in] dev
1602  *   Pointer to rte_eth_dev structure.
1603  * @param[in, out] dev_flow
1604  *   Pointer to the mlx5_flow.
1605  * @param[in] transfer
1606  *   Mark if the flow is E-Switch flow.
1607  * @param[out] error
1608  *   Pointer to the error structure.
1609  *
1610  * @return
1611  *   0 on success, a negative errno value otherwise and rte_errno is set.
1612  */
1613 static int
1614 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1615                                struct mlx5_flow *dev_flow,
1616                                uint8_t transfer,
1617                                struct rte_flow_error *error)
1618 {
1619         struct mlx5_flow_dv_encap_decap_resource res = {
1620                 .size = 0,
1621                 .reformat_type =
1622                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1623                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1624                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1625         };
1626
1627         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1628                 return rte_flow_error_set(error, EINVAL,
1629                                           RTE_FLOW_ERROR_TYPE_ACTION,
1630                                           NULL, "can't create L2 decap action");
1631         return 0;
1632 }
1633
1634 /**
1635  * Convert raw decap/encap (L3 tunnel) action to DV specification.
1636  *
1637  * @param[in] dev
1638  *   Pointer to rte_eth_dev structure.
1639  * @param[in] action
1640  *   Pointer to action structure.
1641  * @param[in, out] dev_flow
1642  *   Pointer to the mlx5_flow.
1643  * @param[in] attr
1644  *   Pointer to the flow attributes.
1645  * @param[out] error
1646  *   Pointer to the error structure.
1647  *
1648  * @return
1649  *   0 on success, a negative errno value otherwise and rte_errno is set.
1650  */
1651 static int
1652 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1653                                 const struct rte_flow_action *action,
1654                                 struct mlx5_flow *dev_flow,
1655                                 const struct rte_flow_attr *attr,
1656                                 struct rte_flow_error *error)
1657 {
1658         const struct rte_flow_action_raw_encap *encap_data;
1659         struct mlx5_flow_dv_encap_decap_resource res;
1660
1661         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1662         res.size = encap_data->size;
1663         memcpy(res.buf, encap_data->data, res.size);
1664         res.reformat_type = attr->egress ?
1665                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1666                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1667         if (attr->transfer)
1668                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1669         else
1670                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1671                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1672         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1673                 return rte_flow_error_set(error, EINVAL,
1674                                           RTE_FLOW_ERROR_TYPE_ACTION,
1675                                           NULL, "can't create encap action");
1676         return 0;
1677 }
1678
1679 /**
1680  * Validate the modify-header actions.
1681  *
1682  * @param[in] action_flags
1683  *   Holds the actions detected until now.
1684  * @param[in] action
1685  *   Pointer to the modify action.
1686  * @param[out] error
1687  *   Pointer to error structure.
1688  *
1689  * @return
1690  *   0 on success, a negative errno value otherwise and rte_errno is set.
1691  */
1692 static int
1693 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1694                                    const struct rte_flow_action *action,
1695                                    struct rte_flow_error *error)
1696 {
1697         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1698                 return rte_flow_error_set(error, EINVAL,
1699                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1700                                           NULL, "action configuration not set");
1701         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1702                 return rte_flow_error_set(error, EINVAL,
1703                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1704                                           "can't have encap action before"
1705                                           " modify action");
1706         return 0;
1707 }
1708
1709 /**
1710  * Validate the modify-header MAC address actions.
1711  *
1712  * @param[in] action_flags
1713  *   Holds the actions detected until now.
1714  * @param[in] action
1715  *   Pointer to the modify action.
1716  * @param[in] item_flags
1717  *   Holds the items detected.
1718  * @param[out] error
1719  *   Pointer to error structure.
1720  *
1721  * @return
1722  *   0 on success, a negative errno value otherwise and rte_errno is set.
1723  */
1724 static int
1725 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1726                                    const struct rte_flow_action *action,
1727                                    const uint64_t item_flags,
1728                                    struct rte_flow_error *error)
1729 {
1730         int ret = 0;
1731
1732         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1733         if (!ret) {
1734                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1735                         return rte_flow_error_set(error, EINVAL,
1736                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1737                                                   NULL,
1738                                                   "no L2 item in pattern");
1739         }
1740         return ret;
1741 }
1742
1743 /**
1744  * Validate the modify-header IPv4 address actions.
1745  *
1746  * @param[in] action_flags
1747  *   Holds the actions detected until now.
1748  * @param[in] action
1749  *   Pointer to the modify action.
1750  * @param[in] item_flags
1751  *   Holds the items detected.
1752  * @param[out] error
1753  *   Pointer to error structure.
1754  *
1755  * @return
1756  *   0 on success, a negative errno value otherwise and rte_errno is set.
1757  */
1758 static int
1759 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
1760                                     const struct rte_flow_action *action,
1761                                     const uint64_t item_flags,
1762                                     struct rte_flow_error *error)
1763 {
1764         int ret = 0;
1765
1766         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1767         if (!ret) {
1768                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
1769                         return rte_flow_error_set(error, EINVAL,
1770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1771                                                   NULL,
1772                                                   "no ipv4 item in pattern");
1773         }
1774         return ret;
1775 }
1776
1777 /**
1778  * Validate the modify-header IPv6 address actions.
1779  *
1780  * @param[in] action_flags
1781  *   Holds the actions detected until now.
1782  * @param[in] action
1783  *   Pointer to the modify action.
1784  * @param[in] item_flags
1785  *   Holds the items detected.
1786  * @param[out] error
1787  *   Pointer to error structure.
1788  *
1789  * @return
1790  *   0 on success, a negative errno value otherwise and rte_errno is set.
1791  */
1792 static int
1793 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
1794                                     const struct rte_flow_action *action,
1795                                     const uint64_t item_flags,
1796                                     struct rte_flow_error *error)
1797 {
1798         int ret = 0;
1799
1800         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1801         if (!ret) {
1802                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
1803                         return rte_flow_error_set(error, EINVAL,
1804                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1805                                                   NULL,
1806                                                   "no ipv6 item in pattern");
1807         }
1808         return ret;
1809 }
1810
1811 /**
1812  * Validate the modify-header TP actions.
1813  *
1814  * @param[in] action_flags
1815  *   Holds the actions detected until now.
1816  * @param[in] action
1817  *   Pointer to the modify action.
1818  * @param[in] item_flags
1819  *   Holds the items detected.
1820  * @param[out] error
1821  *   Pointer to error structure.
1822  *
1823  * @return
1824  *   0 on success, a negative errno value otherwise and rte_errno is set.
1825  */
1826 static int
1827 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
1828                                   const struct rte_flow_action *action,
1829                                   const uint64_t item_flags,
1830                                   struct rte_flow_error *error)
1831 {
1832         int ret = 0;
1833
1834         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1835         if (!ret) {
1836                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
1837                         return rte_flow_error_set(error, EINVAL,
1838                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1839                                                   NULL, "no transport layer "
1840                                                   "in pattern");
1841         }
1842         return ret;
1843 }
1844
1845 /**
1846  * Validate the modify-header actions of increment/decrement
1847  * TCP Sequence-number.
1848  *
1849  * @param[in] action_flags
1850  *   Holds the actions detected until now.
1851  * @param[in] action
1852  *   Pointer to the modify action.
1853  * @param[in] item_flags
1854  *   Holds the items detected.
1855  * @param[out] error
1856  *   Pointer to error structure.
1857  *
1858  * @return
1859  *   0 on success, a negative errno value otherwise and rte_errno is set.
1860  */
1861 static int
1862 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
1863                                        const struct rte_flow_action *action,
1864                                        const uint64_t item_flags,
1865                                        struct rte_flow_error *error)
1866 {
1867         int ret = 0;
1868
1869         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1870         if (!ret) {
1871                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1872                         return rte_flow_error_set(error, EINVAL,
1873                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1874                                                   NULL, "no TCP item in"
1875                                                   " pattern");
1876                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
1877                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
1878                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
1879                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
1880                         return rte_flow_error_set(error, EINVAL,
1881                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1882                                                   NULL,
1883                                                   "cannot decrease and increase"
1884                                                   " TCP sequence number"
1885                                                   " at the same time");
1886         }
1887         return ret;
1888 }
1889
1890 /**
1891  * Validate the modify-header actions of increment/decrement
1892  * TCP Acknowledgment number.
1893  *
1894  * @param[in] action_flags
1895  *   Holds the actions detected until now.
1896  * @param[in] action
1897  *   Pointer to the modify action.
1898  * @param[in] item_flags
1899  *   Holds the items detected.
1900  * @param[out] error
1901  *   Pointer to error structure.
1902  *
1903  * @return
1904  *   0 on success, a negative errno value otherwise and rte_errno is set.
1905  */
1906 static int
1907 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
1908                                        const struct rte_flow_action *action,
1909                                        const uint64_t item_flags,
1910                                        struct rte_flow_error *error)
1911 {
1912         int ret = 0;
1913
1914         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1915         if (!ret) {
1916                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1917                         return rte_flow_error_set(error, EINVAL,
1918                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1919                                                   NULL, "no TCP item in"
1920                                                   " pattern");
1921                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
1922                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
1923                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
1924                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
1925                         return rte_flow_error_set(error, EINVAL,
1926                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1927                                                   NULL,
1928                                                   "cannot decrease and increase"
1929                                                   " TCP acknowledgment number"
1930                                                   " at the same time");
1931         }
1932         return ret;
1933 }
1934
1935 /**
1936  * Validate the modify-header TTL actions.
1937  *
1938  * @param[in] action_flags
1939  *   Holds the actions detected until now.
1940  * @param[in] action
1941  *   Pointer to the modify action.
1942  * @param[in] item_flags
1943  *   Holds the items detected.
1944  * @param[out] error
1945  *   Pointer to error structure.
1946  *
1947  * @return
1948  *   0 on success, a negative errno value otherwise and rte_errno is set.
1949  */
1950 static int
1951 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
1952                                    const struct rte_flow_action *action,
1953                                    const uint64_t item_flags,
1954                                    struct rte_flow_error *error)
1955 {
1956         int ret = 0;
1957
1958         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1959         if (!ret) {
1960                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
1961                         return rte_flow_error_set(error, EINVAL,
1962                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1963                                                   NULL,
1964                                                   "no IP protocol in pattern");
1965         }
1966         return ret;
1967 }
1968
1969 /**
1970  * Validate jump action.
1971  *
1972  * @param[in] action
1973  *   Pointer to the modify action.
1974  * @param[in] group
1975  *   The group of the current flow.
1976  * @param[out] error
1977  *   Pointer to error structure.
1978  *
1979  * @return
1980  *   0 on success, a negative errno value otherwise and rte_errno is set.
1981  */
1982 static int
1983 flow_dv_validate_action_jump(const struct rte_flow_action *action,
1984                              uint32_t group,
1985                              struct rte_flow_error *error)
1986 {
1987         if (action->type != RTE_FLOW_ACTION_TYPE_JUMP && !action->conf)
1988                 return rte_flow_error_set(error, EINVAL,
1989                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1990                                           NULL, "action configuration not set");
1991         if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
1992                 return rte_flow_error_set(error, EINVAL,
1993                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1994                                           "target group must be higher then"
1995                                           " the current flow group");
1996         return 0;
1997 }
1998
1999 /*
2000  * Validate the port_id action.
2001  *
2002  * @param[in] dev
2003  *   Pointer to rte_eth_dev structure.
2004  * @param[in] action_flags
2005  *   Bit-fields that holds the actions detected until now.
2006  * @param[in] action
2007  *   Port_id RTE action structure.
2008  * @param[in] attr
2009  *   Attributes of flow that includes this action.
2010  * @param[out] error
2011  *   Pointer to error structure.
2012  *
2013  * @return
2014  *   0 on success, a negative errno value otherwise and rte_errno is set.
2015  */
2016 static int
2017 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2018                                 uint64_t action_flags,
2019                                 const struct rte_flow_action *action,
2020                                 const struct rte_flow_attr *attr,
2021                                 struct rte_flow_error *error)
2022 {
2023         const struct rte_flow_action_port_id *port_id;
2024         uint16_t port;
2025         uint16_t esw_domain_id;
2026         uint16_t act_port_domain_id;
2027         int ret;
2028
2029         if (!attr->transfer)
2030                 return rte_flow_error_set(error, ENOTSUP,
2031                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2032                                           NULL,
2033                                           "port id action is valid in transfer"
2034                                           " mode only");
2035         if (!action || !action->conf)
2036                 return rte_flow_error_set(error, ENOTSUP,
2037                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2038                                           NULL,
2039                                           "port id action parameters must be"
2040                                           " specified");
2041         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2042                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2043                 return rte_flow_error_set(error, EINVAL,
2044                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2045                                           "can have only one fate actions in"
2046                                           " a flow");
2047         ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2048                                         &esw_domain_id, NULL);
2049         if (ret < 0)
2050                 return rte_flow_error_set(error, -ret,
2051                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2052                                           NULL,
2053                                           "failed to obtain E-Switch info");
2054         port_id = action->conf;
2055         port = port_id->original ? dev->data->port_id : port_id->id;
2056         ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2057         if (ret)
2058                 return rte_flow_error_set
2059                                 (error, -ret,
2060                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2061                                  "failed to obtain E-Switch port id for port");
2062         if (act_port_domain_id != esw_domain_id)
2063                 return rte_flow_error_set
2064                                 (error, -ret,
2065                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2066                                  "port does not belong to"
2067                                  " E-Switch being configured");
2068         return 0;
2069 }
2070
2071 /**
2072  * Find existing modify-header resource or create and register a new one.
2073  *
2074  * @param dev[in, out]
2075  *   Pointer to rte_eth_dev structure.
2076  * @param[in, out] resource
2077  *   Pointer to modify-header resource.
2078  * @parm[in, out] dev_flow
2079  *   Pointer to the dev_flow.
2080  * @param[out] error
2081  *   pointer to error structure.
2082  *
2083  * @return
2084  *   0 on success otherwise -errno and errno is set.
2085  */
2086 static int
2087 flow_dv_modify_hdr_resource_register
2088                         (struct rte_eth_dev *dev,
2089                          struct mlx5_flow_dv_modify_hdr_resource *resource,
2090                          struct mlx5_flow *dev_flow,
2091                          struct rte_flow_error *error)
2092 {
2093         struct mlx5_priv *priv = dev->data->dev_private;
2094         struct mlx5_ibv_shared *sh = priv->sh;
2095         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2096         struct mlx5dv_dr_domain *ns;
2097
2098         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2099                 ns = sh->fdb_domain;
2100         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2101                 ns = sh->tx_domain;
2102         else
2103                 ns = sh->rx_domain;
2104         resource->flags =
2105                 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2106         /* Lookup a matching resource from cache. */
2107         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2108                 if (resource->ft_type == cache_resource->ft_type &&
2109                     resource->actions_num == cache_resource->actions_num &&
2110                     resource->flags == cache_resource->flags &&
2111                     !memcmp((const void *)resource->actions,
2112                             (const void *)cache_resource->actions,
2113                             (resource->actions_num *
2114                                             sizeof(resource->actions[0])))) {
2115                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2116                                 (void *)cache_resource,
2117                                 rte_atomic32_read(&cache_resource->refcnt));
2118                         rte_atomic32_inc(&cache_resource->refcnt);
2119                         dev_flow->dv.modify_hdr = cache_resource;
2120                         return 0;
2121                 }
2122         }
2123         /* Register new modify-header resource. */
2124         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2125         if (!cache_resource)
2126                 return rte_flow_error_set(error, ENOMEM,
2127                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2128                                           "cannot allocate resource memory");
2129         *cache_resource = *resource;
2130         cache_resource->verbs_action =
2131                 mlx5_glue->dv_create_flow_action_modify_header
2132                                         (sh->ctx, cache_resource->ft_type,
2133                                          ns, cache_resource->flags,
2134                                          cache_resource->actions_num *
2135                                          sizeof(cache_resource->actions[0]),
2136                                          (uint64_t *)cache_resource->actions);
2137         if (!cache_resource->verbs_action) {
2138                 rte_free(cache_resource);
2139                 return rte_flow_error_set(error, ENOMEM,
2140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2141                                           NULL, "cannot create action");
2142         }
2143         rte_atomic32_init(&cache_resource->refcnt);
2144         rte_atomic32_inc(&cache_resource->refcnt);
2145         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2146         dev_flow->dv.modify_hdr = cache_resource;
2147         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2148                 (void *)cache_resource,
2149                 rte_atomic32_read(&cache_resource->refcnt));
2150         return 0;
2151 }
2152
2153 #define MLX5_CNT_CONTAINER_RESIZE 64
2154
2155 /**
2156  * Get or create a flow counter.
2157  *
2158  * @param[in] dev
2159  *   Pointer to the Ethernet device structure.
2160  * @param[in] shared
2161  *   Indicate if this counter is shared with other flows.
2162  * @param[in] id
2163  *   Counter identifier.
2164  *
2165  * @return
2166  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2167  */
2168 static struct mlx5_flow_counter *
2169 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2170                                uint32_t id)
2171 {
2172         struct mlx5_priv *priv = dev->data->dev_private;
2173         struct mlx5_flow_counter *cnt = NULL;
2174         struct mlx5_devx_obj *dcs = NULL;
2175
2176         if (!priv->config.devx) {
2177                 rte_errno = ENOTSUP;
2178                 return NULL;
2179         }
2180         if (shared) {
2181                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2182                         if (cnt->shared && cnt->id == id) {
2183                                 cnt->ref_cnt++;
2184                                 return cnt;
2185                         }
2186                 }
2187         }
2188         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2189         if (!dcs)
2190                 return NULL;
2191         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2192         if (!cnt) {
2193                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2194                 rte_errno = ENOMEM;
2195                 return NULL;
2196         }
2197         struct mlx5_flow_counter tmpl = {
2198                 .shared = shared,
2199                 .ref_cnt = 1,
2200                 .id = id,
2201                 .dcs = dcs,
2202         };
2203         tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2204         if (!tmpl.action) {
2205                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2206                 rte_errno = errno;
2207                 rte_free(cnt);
2208                 return NULL;
2209         }
2210         *cnt = tmpl;
2211         TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2212         return cnt;
2213 }
2214
2215 /**
2216  * Release a flow counter.
2217  *
2218  * @param[in] dev
2219  *   Pointer to the Ethernet device structure.
2220  * @param[in] counter
2221  *   Pointer to the counter handler.
2222  */
2223 static void
2224 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2225                                  struct mlx5_flow_counter *counter)
2226 {
2227         struct mlx5_priv *priv = dev->data->dev_private;
2228
2229         if (!counter)
2230                 return;
2231         if (--counter->ref_cnt == 0) {
2232                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2233                 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2234                 rte_free(counter);
2235         }
2236 }
2237
2238 /**
2239  * Query a devx flow counter.
2240  *
2241  * @param[in] dev
2242  *   Pointer to the Ethernet device structure.
2243  * @param[in] cnt
2244  *   Pointer to the flow counter.
2245  * @param[out] pkts
2246  *   The statistics value of packets.
2247  * @param[out] bytes
2248  *   The statistics value of bytes.
2249  *
2250  * @return
2251  *   0 on success, otherwise a negative errno value and rte_errno is set.
2252  */
2253 static inline int
2254 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2255                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2256                      uint64_t *bytes)
2257 {
2258         return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2259                                                 0, NULL, NULL, 0);
2260 }
2261
2262 /**
2263  * Get a pool by a counter.
2264  *
2265  * @param[in] cnt
2266  *   Pointer to the counter.
2267  *
2268  * @return
2269  *   The counter pool.
2270  */
2271 static struct mlx5_flow_counter_pool *
2272 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2273 {
2274         if (!cnt->batch) {
2275                 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2276                 return (struct mlx5_flow_counter_pool *)cnt - 1;
2277         }
2278         return cnt->pool;
2279 }
2280
2281 /**
2282  * Get a pool by devx counter ID.
2283  *
2284  * @param[in] cont
2285  *   Pointer to the counter container.
2286  * @param[in] id
2287  *   The counter devx ID.
2288  *
2289  * @return
2290  *   The counter pool pointer if exists, NULL otherwise,
2291  */
2292 static struct mlx5_flow_counter_pool *
2293 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2294 {
2295         struct mlx5_flow_counter_pool *pool;
2296
2297         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2298                 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2299                                 MLX5_COUNTERS_PER_POOL;
2300
2301                 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2302                         return pool;
2303         };
2304         return NULL;
2305 }
2306
2307 /**
2308  * Allocate a new memory for the counter values wrapped by all the needed
2309  * management.
2310  *
2311  * @param[in] dev
2312  *   Pointer to the Ethernet device structure.
2313  * @param[in] raws_n
2314  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2315  *
2316  * @return
2317  *   The new memory management pointer on success, otherwise NULL and rte_errno
2318  *   is set.
2319  */
2320 static struct mlx5_counter_stats_mem_mng *
2321 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2322 {
2323         struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2324                                         (dev->data->dev_private))->sh;
2325         struct mlx5_devx_mkey_attr mkey_attr;
2326         struct mlx5_counter_stats_mem_mng *mem_mng;
2327         volatile struct flow_counter_stats *raw_data;
2328         int size = (sizeof(struct flow_counter_stats) *
2329                         MLX5_COUNTERS_PER_POOL +
2330                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2331                         sizeof(struct mlx5_counter_stats_mem_mng);
2332         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2333         int i;
2334
2335         if (!mem) {
2336                 rte_errno = ENOMEM;
2337                 return NULL;
2338         }
2339         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2340         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2341         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2342                                                  IBV_ACCESS_LOCAL_WRITE);
2343         if (!mem_mng->umem) {
2344                 rte_errno = errno;
2345                 rte_free(mem);
2346                 return NULL;
2347         }
2348         mkey_attr.addr = (uintptr_t)mem;
2349         mkey_attr.size = size;
2350         mkey_attr.umem_id = mem_mng->umem->umem_id;
2351         mkey_attr.pd = sh->pdn;
2352         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2353         if (!mem_mng->dm) {
2354                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
2355                 rte_errno = errno;
2356                 rte_free(mem);
2357                 return NULL;
2358         }
2359         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
2360         raw_data = (volatile struct flow_counter_stats *)mem;
2361         for (i = 0; i < raws_n; ++i) {
2362                 mem_mng->raws[i].mem_mng = mem_mng;
2363                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
2364         }
2365         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
2366         return mem_mng;
2367 }
2368
2369 /**
2370  * Resize a counter container.
2371  *
2372  * @param[in] dev
2373  *   Pointer to the Ethernet device structure.
2374  * @param[in] batch
2375  *   Whether the pool is for counter that was allocated by batch command.
2376  *
2377  * @return
2378  *   The new container pointer on success, otherwise NULL and rte_errno is set.
2379  */
2380 static struct mlx5_pools_container *
2381 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
2382 {
2383         struct mlx5_priv *priv = dev->data->dev_private;
2384         struct mlx5_pools_container *cont =
2385                         MLX5_CNT_CONTAINER(priv->sh, batch, 0);
2386         struct mlx5_pools_container *new_cont =
2387                         MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
2388         struct mlx5_counter_stats_mem_mng *mem_mng;
2389         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
2390         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
2391         int i;
2392
2393         if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
2394                 /* The last resize still hasn't detected by the host thread. */
2395                 rte_errno = EAGAIN;
2396                 return NULL;
2397         }
2398         new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
2399         if (!new_cont->pools) {
2400                 rte_errno = ENOMEM;
2401                 return NULL;
2402         }
2403         if (cont->n)
2404                 memcpy(new_cont->pools, cont->pools, cont->n *
2405                        sizeof(struct mlx5_flow_counter_pool *));
2406         mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
2407                 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
2408         if (!mem_mng) {
2409                 rte_free(new_cont->pools);
2410                 return NULL;
2411         }
2412         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
2413                 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
2414                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
2415                                  i, next);
2416         new_cont->n = resize;
2417         rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
2418         TAILQ_INIT(&new_cont->pool_list);
2419         TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
2420         new_cont->init_mem_mng = mem_mng;
2421         rte_cio_wmb();
2422          /* Flip the master container. */
2423         priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
2424         return new_cont;
2425 }
2426
2427 /**
2428  * Query a devx flow counter.
2429  *
2430  * @param[in] dev
2431  *   Pointer to the Ethernet device structure.
2432  * @param[in] cnt
2433  *   Pointer to the flow counter.
2434  * @param[out] pkts
2435  *   The statistics value of packets.
2436  * @param[out] bytes
2437  *   The statistics value of bytes.
2438  *
2439  * @return
2440  *   0 on success, otherwise a negative errno value and rte_errno is set.
2441  */
2442 static inline int
2443 _flow_dv_query_count(struct rte_eth_dev *dev,
2444                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2445                      uint64_t *bytes)
2446 {
2447         struct mlx5_priv *priv = dev->data->dev_private;
2448         struct mlx5_flow_counter_pool *pool =
2449                         flow_dv_counter_pool_get(cnt);
2450         int offset = cnt - &pool->counters_raw[0];
2451
2452         if (priv->counter_fallback)
2453                 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
2454
2455         rte_spinlock_lock(&pool->sl);
2456         /*
2457          * The single counters allocation may allocate smaller ID than the
2458          * current allocated in parallel to the host reading.
2459          * In this case the new counter values must be reported as 0.
2460          */
2461         if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
2462                 *pkts = 0;
2463                 *bytes = 0;
2464         } else {
2465                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
2466                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
2467         }
2468         rte_spinlock_unlock(&pool->sl);
2469         return 0;
2470 }
2471
2472 /**
2473  * Create and initialize a new counter pool.
2474  *
2475  * @param[in] dev
2476  *   Pointer to the Ethernet device structure.
2477  * @param[out] dcs
2478  *   The devX counter handle.
2479  * @param[in] batch
2480  *   Whether the pool is for counter that was allocated by batch command.
2481  *
2482  * @return
2483  *   A new pool pointer on success, NULL otherwise and rte_errno is set.
2484  */
2485 static struct mlx5_flow_counter_pool *
2486 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
2487                     uint32_t batch)
2488 {
2489         struct mlx5_priv *priv = dev->data->dev_private;
2490         struct mlx5_flow_counter_pool *pool;
2491         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2492                                                                0);
2493         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
2494         uint32_t size;
2495
2496         if (cont->n == n_valid) {
2497                 cont = flow_dv_container_resize(dev, batch);
2498                 if (!cont)
2499                         return NULL;
2500         }
2501         size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
2502                         sizeof(struct mlx5_flow_counter);
2503         pool = rte_calloc(__func__, 1, size, 0);
2504         if (!pool) {
2505                 rte_errno = ENOMEM;
2506                 return NULL;
2507         }
2508         pool->min_dcs = dcs;
2509         pool->raw = cont->init_mem_mng->raws + n_valid %
2510                                                      MLX5_CNT_CONTAINER_RESIZE;
2511         pool->raw_hw = NULL;
2512         rte_spinlock_init(&pool->sl);
2513         /*
2514          * The generation of the new allocated counters in this pool is 0, 2 in
2515          * the pool generation makes all the counters valid for allocation.
2516          */
2517         rte_atomic64_set(&pool->query_gen, 0x2);
2518         TAILQ_INIT(&pool->counters);
2519         TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2520         cont->pools[n_valid] = pool;
2521         /* Pool initialization must be updated before host thread access. */
2522         rte_cio_wmb();
2523         rte_atomic16_add(&cont->n_valid, 1);
2524         return pool;
2525 }
2526
2527 /**
2528  * Prepare a new counter and/or a new counter pool.
2529  *
2530  * @param[in] dev
2531  *   Pointer to the Ethernet device structure.
2532  * @param[out] cnt_free
2533  *   Where to put the pointer of a new counter.
2534  * @param[in] batch
2535  *   Whether the pool is for counter that was allocated by batch command.
2536  *
2537  * @return
2538  *   The free counter pool pointer and @p cnt_free is set on success,
2539  *   NULL otherwise and rte_errno is set.
2540  */
2541 static struct mlx5_flow_counter_pool *
2542 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
2543                              struct mlx5_flow_counter **cnt_free,
2544                              uint32_t batch)
2545 {
2546         struct mlx5_priv *priv = dev->data->dev_private;
2547         struct mlx5_flow_counter_pool *pool;
2548         struct mlx5_devx_obj *dcs = NULL;
2549         struct mlx5_flow_counter *cnt;
2550         uint32_t i;
2551
2552         if (!batch) {
2553                 /* bulk_bitmap must be 0 for single counter allocation. */
2554                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2555                 if (!dcs)
2556                         return NULL;
2557                 pool = flow_dv_find_pool_by_id
2558                         (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
2559                 if (!pool) {
2560                         pool = flow_dv_pool_create(dev, dcs, batch);
2561                         if (!pool) {
2562                                 mlx5_devx_cmd_destroy(dcs);
2563                                 return NULL;
2564                         }
2565                 } else if (dcs->id < pool->min_dcs->id) {
2566                         rte_atomic64_set(&pool->a64_dcs,
2567                                          (int64_t)(uintptr_t)dcs);
2568                 }
2569                 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
2570                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2571                 cnt->dcs = dcs;
2572                 *cnt_free = cnt;
2573                 return pool;
2574         }
2575         /* bulk_bitmap is in 128 counters units. */
2576         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
2577                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
2578         if (!dcs) {
2579                 rte_errno = ENODATA;
2580                 return NULL;
2581         }
2582         pool = flow_dv_pool_create(dev, dcs, batch);
2583         if (!pool) {
2584                 mlx5_devx_cmd_destroy(dcs);
2585                 return NULL;
2586         }
2587         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2588                 cnt = &pool->counters_raw[i];
2589                 cnt->pool = pool;
2590                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2591         }
2592         *cnt_free = &pool->counters_raw[0];
2593         return pool;
2594 }
2595
2596 /**
2597  * Search for existed shared counter.
2598  *
2599  * @param[in] cont
2600  *   Pointer to the relevant counter pool container.
2601  * @param[in] id
2602  *   The shared counter ID to search.
2603  *
2604  * @return
2605  *   NULL if not existed, otherwise pointer to the shared counter.
2606  */
2607 static struct mlx5_flow_counter *
2608 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
2609                               uint32_t id)
2610 {
2611         static struct mlx5_flow_counter *cnt;
2612         struct mlx5_flow_counter_pool *pool;
2613         int i;
2614
2615         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2616                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2617                         cnt = &pool->counters_raw[i];
2618                         if (cnt->ref_cnt && cnt->shared && cnt->id == id)
2619                                 return cnt;
2620                 }
2621         }
2622         return NULL;
2623 }
2624
2625 /**
2626  * Allocate a flow counter.
2627  *
2628  * @param[in] dev
2629  *   Pointer to the Ethernet device structure.
2630  * @param[in] shared
2631  *   Indicate if this counter is shared with other flows.
2632  * @param[in] id
2633  *   Counter identifier.
2634  * @param[in] group
2635  *   Counter flow group.
2636  *
2637  * @return
2638  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2639  */
2640 static struct mlx5_flow_counter *
2641 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
2642                       uint16_t group)
2643 {
2644         struct mlx5_priv *priv = dev->data->dev_private;
2645         struct mlx5_flow_counter_pool *pool = NULL;
2646         struct mlx5_flow_counter *cnt_free = NULL;
2647         /*
2648          * Currently group 0 flow counter cannot be assigned to a flow if it is
2649          * not the first one in the batch counter allocation, so it is better
2650          * to allocate counters one by one for these flows in a separate
2651          * container.
2652          * A counter can be shared between different groups so need to take
2653          * shared counters from the single container.
2654          */
2655         uint32_t batch = (group && !shared) ? 1 : 0;
2656         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2657                                                                0);
2658
2659         if (priv->counter_fallback)
2660                 return flow_dv_counter_alloc_fallback(dev, shared, id);
2661         if (!priv->config.devx) {
2662                 rte_errno = ENOTSUP;
2663                 return NULL;
2664         }
2665         if (shared) {
2666                 cnt_free = flow_dv_counter_shared_search(cont, id);
2667                 if (cnt_free) {
2668                         if (cnt_free->ref_cnt + 1 == 0) {
2669                                 rte_errno = E2BIG;
2670                                 return NULL;
2671                         }
2672                         cnt_free->ref_cnt++;
2673                         return cnt_free;
2674                 }
2675         }
2676         /* Pools which has a free counters are in the start. */
2677         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2678                 /*
2679                  * The free counter reset values must be updated between the
2680                  * counter release to the counter allocation, so, at least one
2681                  * query must be done in this time. ensure it by saving the
2682                  * query generation in the release time.
2683                  * The free list is sorted according to the generation - so if
2684                  * the first one is not updated, all the others are not
2685                  * updated too.
2686                  */
2687                 cnt_free = TAILQ_FIRST(&pool->counters);
2688                 if (cnt_free && cnt_free->query_gen + 1 <
2689                     rte_atomic64_read(&pool->query_gen))
2690                         break;
2691                 cnt_free = NULL;
2692         }
2693         if (!cnt_free) {
2694                 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
2695                 if (!pool)
2696                         return NULL;
2697         }
2698         cnt_free->batch = batch;
2699         /* Create a DV counter action only in the first time usage. */
2700         if (!cnt_free->action) {
2701                 uint16_t offset;
2702                 struct mlx5_devx_obj *dcs;
2703
2704                 if (batch) {
2705                         offset = cnt_free - &pool->counters_raw[0];
2706                         dcs = pool->min_dcs;
2707                 } else {
2708                         offset = 0;
2709                         dcs = cnt_free->dcs;
2710                 }
2711                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
2712                                         (dcs->obj, offset);
2713                 if (!cnt_free->action) {
2714                         rte_errno = errno;
2715                         return NULL;
2716                 }
2717         }
2718         /* Update the counter reset values. */
2719         if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
2720                                  &cnt_free->bytes))
2721                 return NULL;
2722         cnt_free->shared = shared;
2723         cnt_free->ref_cnt = 1;
2724         cnt_free->id = id;
2725         if (!priv->sh->cmng.query_thread_on)
2726                 /* Start the asynchronous batch query by the host thread. */
2727                 mlx5_set_query_alarm(priv->sh);
2728         TAILQ_REMOVE(&pool->counters, cnt_free, next);
2729         if (TAILQ_EMPTY(&pool->counters)) {
2730                 /* Move the pool to the end of the container pool list. */
2731                 TAILQ_REMOVE(&cont->pool_list, pool, next);
2732                 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2733         }
2734         return cnt_free;
2735 }
2736
2737 /**
2738  * Release a flow counter.
2739  *
2740  * @param[in] dev
2741  *   Pointer to the Ethernet device structure.
2742  * @param[in] counter
2743  *   Pointer to the counter handler.
2744  */
2745 static void
2746 flow_dv_counter_release(struct rte_eth_dev *dev,
2747                         struct mlx5_flow_counter *counter)
2748 {
2749         struct mlx5_priv *priv = dev->data->dev_private;
2750
2751         if (!counter)
2752                 return;
2753         if (priv->counter_fallback) {
2754                 flow_dv_counter_release_fallback(dev, counter);
2755                 return;
2756         }
2757         if (--counter->ref_cnt == 0) {
2758                 struct mlx5_flow_counter_pool *pool =
2759                                 flow_dv_counter_pool_get(counter);
2760
2761                 /* Put the counter in the end - the last updated one. */
2762                 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
2763                 counter->query_gen = rte_atomic64_read(&pool->query_gen);
2764         }
2765 }
2766
2767 /**
2768  * Verify the @p attributes will be correctly understood by the NIC and store
2769  * them in the @p flow if everything is correct.
2770  *
2771  * @param[in] dev
2772  *   Pointer to dev struct.
2773  * @param[in] attributes
2774  *   Pointer to flow attributes
2775  * @param[out] error
2776  *   Pointer to error structure.
2777  *
2778  * @return
2779  *   0 on success, a negative errno value otherwise and rte_errno is set.
2780  */
2781 static int
2782 flow_dv_validate_attributes(struct rte_eth_dev *dev,
2783                             const struct rte_flow_attr *attributes,
2784                             struct rte_flow_error *error)
2785 {
2786         struct mlx5_priv *priv = dev->data->dev_private;
2787         uint32_t priority_max = priv->config.flow_prio - 1;
2788
2789 #ifndef HAVE_MLX5DV_DR
2790         if (attributes->group)
2791                 return rte_flow_error_set(error, ENOTSUP,
2792                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2793                                           NULL,
2794                                           "groups is not supported");
2795 #endif
2796         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
2797             attributes->priority >= priority_max)
2798                 return rte_flow_error_set(error, ENOTSUP,
2799                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2800                                           NULL,
2801                                           "priority out of range");
2802         if (attributes->transfer) {
2803                 if (!priv->config.dv_esw_en)
2804                         return rte_flow_error_set
2805                                 (error, ENOTSUP,
2806                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2807                                  "E-Switch dr is not supported");
2808                 if (!(priv->representor || priv->master))
2809                         return rte_flow_error_set
2810                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2811                                  NULL, "E-Switch configurationd can only be"
2812                                  " done by a master or a representor device");
2813                 if (attributes->egress)
2814                         return rte_flow_error_set
2815                                 (error, ENOTSUP,
2816                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
2817                                  "egress is not supported");
2818                 if (attributes->group >= MLX5_MAX_TABLES_FDB)
2819                         return rte_flow_error_set
2820                                 (error, EINVAL,
2821                                  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2822                                  NULL, "group must be smaller than "
2823                                  RTE_STR(MLX5_MAX_FDB_TABLES));
2824         }
2825         if (!(attributes->egress ^ attributes->ingress))
2826                 return rte_flow_error_set(error, ENOTSUP,
2827                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
2828                                           "must specify exactly one of "
2829                                           "ingress or egress");
2830         return 0;
2831 }
2832
2833 /**
2834  * Internal validation function. For validating both actions and items.
2835  *
2836  * @param[in] dev
2837  *   Pointer to the rte_eth_dev structure.
2838  * @param[in] attr
2839  *   Pointer to the flow attributes.
2840  * @param[in] items
2841  *   Pointer to the list of items.
2842  * @param[in] actions
2843  *   Pointer to the list of actions.
2844  * @param[out] error
2845  *   Pointer to the error structure.
2846  *
2847  * @return
2848  *   0 on success, a negative errno value otherwise and rte_errno is set.
2849  */
2850 static int
2851 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
2852                  const struct rte_flow_item items[],
2853                  const struct rte_flow_action actions[],
2854                  struct rte_flow_error *error)
2855 {
2856         int ret;
2857         uint64_t action_flags = 0;
2858         uint64_t item_flags = 0;
2859         uint64_t last_item = 0;
2860         uint8_t next_protocol = 0xff;
2861         int actions_n = 0;
2862         const struct rte_flow_item *gre_item = NULL;
2863         struct rte_flow_item_tcp nic_tcp_mask = {
2864                 .hdr = {
2865                         .tcp_flags = 0xFF,
2866                         .src_port = RTE_BE16(UINT16_MAX),
2867                         .dst_port = RTE_BE16(UINT16_MAX),
2868                 }
2869         };
2870
2871         if (items == NULL)
2872                 return -1;
2873         ret = flow_dv_validate_attributes(dev, attr, error);
2874         if (ret < 0)
2875                 return ret;
2876         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2877                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2878                 switch (items->type) {
2879                 case RTE_FLOW_ITEM_TYPE_VOID:
2880                         break;
2881                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2882                         ret = flow_dv_validate_item_port_id
2883                                         (dev, items, attr, item_flags, error);
2884                         if (ret < 0)
2885                                 return ret;
2886                         last_item = MLX5_FLOW_ITEM_PORT_ID;
2887                         break;
2888                 case RTE_FLOW_ITEM_TYPE_ETH:
2889                         ret = mlx5_flow_validate_item_eth(items, item_flags,
2890                                                           error);
2891                         if (ret < 0)
2892                                 return ret;
2893                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2894                                              MLX5_FLOW_LAYER_OUTER_L2;
2895                         break;
2896                 case RTE_FLOW_ITEM_TYPE_VLAN:
2897                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2898                                                            error);
2899                         if (ret < 0)
2900                                 return ret;
2901                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2902                                              MLX5_FLOW_LAYER_OUTER_VLAN;
2903                         break;
2904                 case RTE_FLOW_ITEM_TYPE_IPV4:
2905                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2906                                                            NULL, error);
2907                         if (ret < 0)
2908                                 return ret;
2909                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2910                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2911                         if (items->mask != NULL &&
2912                             ((const struct rte_flow_item_ipv4 *)
2913                              items->mask)->hdr.next_proto_id) {
2914                                 next_protocol =
2915                                         ((const struct rte_flow_item_ipv4 *)
2916                                          (items->spec))->hdr.next_proto_id;
2917                                 next_protocol &=
2918                                         ((const struct rte_flow_item_ipv4 *)
2919                                          (items->mask))->hdr.next_proto_id;
2920                         } else {
2921                                 /* Reset for inner layer. */
2922                                 next_protocol = 0xff;
2923                         }
2924                         mlx5_flow_tunnel_ip_check(items, &last_item);
2925                         break;
2926                 case RTE_FLOW_ITEM_TYPE_IPV6:
2927                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2928                                                            NULL, error);
2929                         if (ret < 0)
2930                                 return ret;
2931                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2932                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2933                         if (items->mask != NULL &&
2934                             ((const struct rte_flow_item_ipv6 *)
2935                              items->mask)->hdr.proto) {
2936                                 next_protocol =
2937                                         ((const struct rte_flow_item_ipv6 *)
2938                                          items->spec)->hdr.proto;
2939                                 next_protocol &=
2940                                         ((const struct rte_flow_item_ipv6 *)
2941                                          items->mask)->hdr.proto;
2942                         } else {
2943                                 /* Reset for inner layer. */
2944                                 next_protocol = 0xff;
2945                         }
2946                         mlx5_flow_tunnel_ip_check(items, &last_item);
2947                         break;
2948                 case RTE_FLOW_ITEM_TYPE_TCP:
2949                         ret = mlx5_flow_validate_item_tcp
2950                                                 (items, item_flags,
2951                                                  next_protocol,
2952                                                  &nic_tcp_mask,
2953                                                  error);
2954                         if (ret < 0)
2955                                 return ret;
2956                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
2957                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
2958                         break;
2959                 case RTE_FLOW_ITEM_TYPE_UDP:
2960                         ret = mlx5_flow_validate_item_udp(items, item_flags,
2961                                                           next_protocol,
2962                                                           error);
2963                         if (ret < 0)
2964                                 return ret;
2965                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
2966                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
2967                         break;
2968                 case RTE_FLOW_ITEM_TYPE_GRE:
2969                         ret = mlx5_flow_validate_item_gre(items, item_flags,
2970                                                           next_protocol, error);
2971                         if (ret < 0)
2972                                 return ret;
2973                         gre_item = items;
2974                         last_item = MLX5_FLOW_LAYER_GRE;
2975                         break;
2976                 case RTE_FLOW_ITEM_TYPE_NVGRE:
2977                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
2978                                                             next_protocol,
2979                                                             error);
2980                         if (ret < 0)
2981                                 return ret;
2982                         last_item = MLX5_FLOW_LAYER_NVGRE;
2983                         break;
2984                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
2985                         ret = mlx5_flow_validate_item_gre_key
2986                                 (items, item_flags, gre_item, error);
2987                         if (ret < 0)
2988                                 return ret;
2989                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
2990                         break;
2991                 case RTE_FLOW_ITEM_TYPE_VXLAN:
2992                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
2993                                                             error);
2994                         if (ret < 0)
2995                                 return ret;
2996                         last_item = MLX5_FLOW_LAYER_VXLAN;
2997                         break;
2998                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2999                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
3000                                                                 item_flags, dev,
3001                                                                 error);
3002                         if (ret < 0)
3003                                 return ret;
3004                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3005                         break;
3006                 case RTE_FLOW_ITEM_TYPE_MPLS:
3007                         ret = mlx5_flow_validate_item_mpls(dev, items,
3008                                                            item_flags,
3009                                                            last_item, error);
3010                         if (ret < 0)
3011                                 return ret;
3012                         last_item = MLX5_FLOW_LAYER_MPLS;
3013                         break;
3014                 case RTE_FLOW_ITEM_TYPE_META:
3015                         ret = flow_dv_validate_item_meta(dev, items, attr,
3016                                                          error);
3017                         if (ret < 0)
3018                                 return ret;
3019                         last_item = MLX5_FLOW_ITEM_METADATA;
3020                         break;
3021                 case RTE_FLOW_ITEM_TYPE_ICMP:
3022                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
3023                                                            next_protocol,
3024                                                            error);
3025                         if (ret < 0)
3026                                 return ret;
3027                         last_item = MLX5_FLOW_LAYER_ICMP;
3028                         break;
3029                 case RTE_FLOW_ITEM_TYPE_ICMP6:
3030                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3031                                                             next_protocol,
3032                                                             error);
3033                         if (ret < 0)
3034                                 return ret;
3035                         last_item = MLX5_FLOW_LAYER_ICMP6;
3036                         break;
3037                 default:
3038                         return rte_flow_error_set(error, ENOTSUP,
3039                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3040                                                   NULL, "item not supported");
3041                 }
3042                 item_flags |= last_item;
3043         }
3044         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3045                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3046                         return rte_flow_error_set(error, ENOTSUP,
3047                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3048                                                   actions, "too many actions");
3049                 switch (actions->type) {
3050                 case RTE_FLOW_ACTION_TYPE_VOID:
3051                         break;
3052                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3053                         ret = flow_dv_validate_action_port_id(dev,
3054                                                               action_flags,
3055                                                               actions,
3056                                                               attr,
3057                                                               error);
3058                         if (ret)
3059                                 return ret;
3060                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3061                         ++actions_n;
3062                         break;
3063                 case RTE_FLOW_ACTION_TYPE_FLAG:
3064                         ret = mlx5_flow_validate_action_flag(action_flags,
3065                                                              attr, error);
3066                         if (ret < 0)
3067                                 return ret;
3068                         action_flags |= MLX5_FLOW_ACTION_FLAG;
3069                         ++actions_n;
3070                         break;
3071                 case RTE_FLOW_ACTION_TYPE_MARK:
3072                         ret = mlx5_flow_validate_action_mark(actions,
3073                                                              action_flags,
3074                                                              attr, error);
3075                         if (ret < 0)
3076                                 return ret;
3077                         action_flags |= MLX5_FLOW_ACTION_MARK;
3078                         ++actions_n;
3079                         break;
3080                 case RTE_FLOW_ACTION_TYPE_DROP:
3081                         ret = mlx5_flow_validate_action_drop(action_flags,
3082                                                              attr, error);
3083                         if (ret < 0)
3084                                 return ret;
3085                         action_flags |= MLX5_FLOW_ACTION_DROP;
3086                         ++actions_n;
3087                         break;
3088                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3089                         ret = mlx5_flow_validate_action_queue(actions,
3090                                                               action_flags, dev,
3091                                                               attr, error);
3092                         if (ret < 0)
3093                                 return ret;
3094                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
3095                         ++actions_n;
3096                         break;
3097                 case RTE_FLOW_ACTION_TYPE_RSS:
3098                         ret = mlx5_flow_validate_action_rss(actions,
3099                                                             action_flags, dev,
3100                                                             attr, item_flags,
3101                                                             error);
3102                         if (ret < 0)
3103                                 return ret;
3104                         action_flags |= MLX5_FLOW_ACTION_RSS;
3105                         ++actions_n;
3106                         break;
3107                 case RTE_FLOW_ACTION_TYPE_COUNT:
3108                         ret = flow_dv_validate_action_count(dev, error);
3109                         if (ret < 0)
3110                                 return ret;
3111                         action_flags |= MLX5_FLOW_ACTION_COUNT;
3112                         ++actions_n;
3113                         break;
3114                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3115                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3116                         ret = flow_dv_validate_action_l2_encap(action_flags,
3117                                                                actions, attr,
3118                                                                error);
3119                         if (ret < 0)
3120                                 return ret;
3121                         action_flags |= actions->type ==
3122                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3123                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
3124                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
3125                         ++actions_n;
3126                         break;
3127                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3128                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3129                         ret = flow_dv_validate_action_l2_decap(action_flags,
3130                                                                attr, error);
3131                         if (ret < 0)
3132                                 return ret;
3133                         action_flags |= actions->type ==
3134                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3135                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
3136                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
3137                         ++actions_n;
3138                         break;
3139                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3140                         ret = flow_dv_validate_action_raw_encap(action_flags,
3141                                                                 actions, attr,
3142                                                                 error);
3143                         if (ret < 0)
3144                                 return ret;
3145                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3146                         ++actions_n;
3147                         break;
3148                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3149                         ret = flow_dv_validate_action_raw_decap(action_flags,
3150                                                                 actions, attr,
3151                                                                 error);
3152                         if (ret < 0)
3153                                 return ret;
3154                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3155                         ++actions_n;
3156                         break;
3157                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3158                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3159                         ret = flow_dv_validate_action_modify_mac(action_flags,
3160                                                                  actions,
3161                                                                  item_flags,
3162                                                                  error);
3163                         if (ret < 0)
3164                                 return ret;
3165                         /* Count all modify-header actions as one action. */
3166                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3167                                 ++actions_n;
3168                         action_flags |= actions->type ==
3169                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3170                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
3171                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
3172                         break;
3173
3174                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3175                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3176                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
3177                                                                   actions,
3178                                                                   item_flags,
3179                                                                   error);
3180                         if (ret < 0)
3181                                 return ret;
3182                         /* Count all modify-header actions as one action. */
3183                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3184                                 ++actions_n;
3185                         action_flags |= actions->type ==
3186                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3187                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3188                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
3189                         break;
3190                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3191                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3192                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
3193                                                                   actions,
3194                                                                   item_flags,
3195                                                                   error);
3196                         if (ret < 0)
3197                                 return ret;
3198                         /* Count all modify-header actions as one action. */
3199                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3200                                 ++actions_n;
3201                         action_flags |= actions->type ==
3202                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3203                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3204                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
3205                         break;
3206                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3207                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3208                         ret = flow_dv_validate_action_modify_tp(action_flags,
3209                                                                 actions,
3210                                                                 item_flags,
3211                                                                 error);
3212                         if (ret < 0)
3213                                 return ret;
3214                         /* Count all modify-header actions as one action. */
3215                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3216                                 ++actions_n;
3217                         action_flags |= actions->type ==
3218                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3219                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
3220                                                 MLX5_FLOW_ACTION_SET_TP_DST;
3221                         break;
3222                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3223                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3224                         ret = flow_dv_validate_action_modify_ttl(action_flags,
3225                                                                  actions,
3226                                                                  item_flags,
3227                                                                  error);
3228                         if (ret < 0)
3229                                 return ret;
3230                         /* Count all modify-header actions as one action. */
3231                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3232                                 ++actions_n;
3233                         action_flags |= actions->type ==
3234                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
3235                                                 MLX5_FLOW_ACTION_SET_TTL :
3236                                                 MLX5_FLOW_ACTION_DEC_TTL;
3237                         break;
3238                 case RTE_FLOW_ACTION_TYPE_JUMP:
3239                         ret = flow_dv_validate_action_jump(actions,
3240                                                            attr->group, error);
3241                         if (ret)
3242                                 return ret;
3243                         ++actions_n;
3244                         action_flags |= MLX5_FLOW_ACTION_JUMP;
3245                         break;
3246                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3247                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3248                         ret = flow_dv_validate_action_modify_tcp_seq
3249                                                                 (action_flags,
3250                                                                  actions,
3251                                                                  item_flags,
3252                                                                  error);
3253                         if (ret < 0)
3254                                 return ret;
3255                         /* Count all modify-header actions as one action. */
3256                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3257                                 ++actions_n;
3258                         action_flags |= actions->type ==
3259                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
3260                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
3261                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
3262                         break;
3263                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3264                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3265                         ret = flow_dv_validate_action_modify_tcp_ack
3266                                                                 (action_flags,
3267                                                                  actions,
3268                                                                  item_flags,
3269                                                                  error);
3270                         if (ret < 0)
3271                                 return ret;
3272                         /* Count all modify-header actions as one action. */
3273                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3274                                 ++actions_n;
3275                         action_flags |= actions->type ==
3276                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
3277                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
3278                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
3279                         break;
3280                 default:
3281                         return rte_flow_error_set(error, ENOTSUP,
3282                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3283                                                   actions,
3284                                                   "action not supported");
3285                 }
3286         }
3287         /* Eswitch has few restrictions on using items and actions */
3288         if (attr->transfer) {
3289                 if (action_flags & MLX5_FLOW_ACTION_FLAG)
3290                         return rte_flow_error_set(error, ENOTSUP,
3291                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3292                                                   NULL,
3293                                                   "unsupported action FLAG");
3294                 if (action_flags & MLX5_FLOW_ACTION_MARK)
3295                         return rte_flow_error_set(error, ENOTSUP,
3296                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3297                                                   NULL,
3298                                                   "unsupported action MARK");
3299                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
3300                         return rte_flow_error_set(error, ENOTSUP,
3301                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3302                                                   NULL,
3303                                                   "unsupported action QUEUE");
3304                 if (action_flags & MLX5_FLOW_ACTION_RSS)
3305                         return rte_flow_error_set(error, ENOTSUP,
3306                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3307                                                   NULL,
3308                                                   "unsupported action RSS");
3309                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3310                         return rte_flow_error_set(error, EINVAL,
3311                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3312                                                   actions,
3313                                                   "no fate action is found");
3314         } else {
3315                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
3316                         return rte_flow_error_set(error, EINVAL,
3317                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3318                                                   actions,
3319                                                   "no fate action is found");
3320         }
3321         return 0;
3322 }
3323
3324 /**
3325  * Internal preparation function. Allocates the DV flow size,
3326  * this size is constant.
3327  *
3328  * @param[in] attr
3329  *   Pointer to the flow attributes.
3330  * @param[in] items
3331  *   Pointer to the list of items.
3332  * @param[in] actions
3333  *   Pointer to the list of actions.
3334  * @param[out] error
3335  *   Pointer to the error structure.
3336  *
3337  * @return
3338  *   Pointer to mlx5_flow object on success,
3339  *   otherwise NULL and rte_errno is set.
3340  */
3341 static struct mlx5_flow *
3342 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
3343                 const struct rte_flow_item items[] __rte_unused,
3344                 const struct rte_flow_action actions[] __rte_unused,
3345                 struct rte_flow_error *error)
3346 {
3347         uint32_t size = sizeof(struct mlx5_flow);
3348         struct mlx5_flow *flow;
3349
3350         flow = rte_calloc(__func__, 1, size, 0);
3351         if (!flow) {
3352                 rte_flow_error_set(error, ENOMEM,
3353                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3354                                    "not enough memory to create flow");
3355                 return NULL;
3356         }
3357         flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
3358         return flow;
3359 }
3360
3361 #ifndef NDEBUG
3362 /**
3363  * Sanity check for match mask and value. Similar to check_valid_spec() in
3364  * kernel driver. If unmasked bit is present in value, it returns failure.
3365  *
3366  * @param match_mask
3367  *   pointer to match mask buffer.
3368  * @param match_value
3369  *   pointer to match value buffer.
3370  *
3371  * @return
3372  *   0 if valid, -EINVAL otherwise.
3373  */
3374 static int
3375 flow_dv_check_valid_spec(void *match_mask, void *match_value)
3376 {
3377         uint8_t *m = match_mask;
3378         uint8_t *v = match_value;
3379         unsigned int i;
3380
3381         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
3382                 if (v[i] & ~m[i]) {
3383                         DRV_LOG(ERR,
3384                                 "match_value differs from match_criteria"
3385                                 " %p[%u] != %p[%u]",
3386                                 match_value, i, match_mask, i);
3387                         return -EINVAL;
3388                 }
3389         }
3390         return 0;
3391 }
3392 #endif
3393
3394 /**
3395  * Add Ethernet item to matcher and to the value.
3396  *
3397  * @param[in, out] matcher
3398  *   Flow matcher.
3399  * @param[in, out] key
3400  *   Flow matcher value.
3401  * @param[in] item
3402  *   Flow pattern to translate.
3403  * @param[in] inner
3404  *   Item is inner pattern.
3405  */
3406 static void
3407 flow_dv_translate_item_eth(void *matcher, void *key,
3408                            const struct rte_flow_item *item, int inner)
3409 {
3410         const struct rte_flow_item_eth *eth_m = item->mask;
3411         const struct rte_flow_item_eth *eth_v = item->spec;
3412         const struct rte_flow_item_eth nic_mask = {
3413                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3414                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3415                 .type = RTE_BE16(0xffff),
3416         };
3417         void *headers_m;
3418         void *headers_v;
3419         char *l24_v;
3420         unsigned int i;
3421
3422         if (!eth_v)
3423                 return;
3424         if (!eth_m)
3425                 eth_m = &nic_mask;
3426         if (inner) {
3427                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3428                                          inner_headers);
3429                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3430         } else {
3431                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3432                                          outer_headers);
3433                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3434         }
3435         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
3436                &eth_m->dst, sizeof(eth_m->dst));
3437         /* The value must be in the range of the mask. */
3438         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
3439         for (i = 0; i < sizeof(eth_m->dst); ++i)
3440                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
3441         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
3442                &eth_m->src, sizeof(eth_m->src));
3443         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
3444         /* The value must be in the range of the mask. */
3445         for (i = 0; i < sizeof(eth_m->dst); ++i)
3446                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
3447         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3448                  rte_be_to_cpu_16(eth_m->type));
3449         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
3450         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
3451 }
3452
3453 /**
3454  * Add VLAN item to matcher and to the value.
3455  *
3456  * @param[in, out] matcher
3457  *   Flow matcher.
3458  * @param[in, out] key
3459  *   Flow matcher value.
3460  * @param[in] item
3461  *   Flow pattern to translate.
3462  * @param[in] inner
3463  *   Item is inner pattern.
3464  */
3465 static void
3466 flow_dv_translate_item_vlan(void *matcher, void *key,
3467                             const struct rte_flow_item *item,
3468                             int inner)
3469 {
3470         const struct rte_flow_item_vlan *vlan_m = item->mask;
3471         const struct rte_flow_item_vlan *vlan_v = item->spec;
3472         const struct rte_flow_item_vlan nic_mask = {
3473                 .tci = RTE_BE16(0x0fff),
3474                 .inner_type = RTE_BE16(0xffff),
3475         };
3476         void *headers_m;
3477         void *headers_v;
3478         uint16_t tci_m;
3479         uint16_t tci_v;
3480
3481         if (!vlan_v)
3482                 return;
3483         if (!vlan_m)
3484                 vlan_m = &nic_mask;
3485         if (inner) {
3486                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3487                                          inner_headers);
3488                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3489         } else {
3490                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3491                                          outer_headers);
3492                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3493         }
3494         tci_m = rte_be_to_cpu_16(vlan_m->tci);
3495         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
3496         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
3497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
3498         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
3499         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
3500         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
3501         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
3502         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
3503         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
3504 }
3505
3506 /**
3507  * Add IPV4 item to matcher and to the value.
3508  *
3509  * @param[in, out] matcher
3510  *   Flow matcher.
3511  * @param[in, out] key
3512  *   Flow matcher value.
3513  * @param[in] item
3514  *   Flow pattern to translate.
3515  * @param[in] inner
3516  *   Item is inner pattern.
3517  * @param[in] group
3518  *   The group to insert the rule.
3519  */
3520 static void
3521 flow_dv_translate_item_ipv4(void *matcher, void *key,
3522                             const struct rte_flow_item *item,
3523                             int inner, uint32_t group)
3524 {
3525         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
3526         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
3527         const struct rte_flow_item_ipv4 nic_mask = {
3528                 .hdr = {
3529                         .src_addr = RTE_BE32(0xffffffff),
3530                         .dst_addr = RTE_BE32(0xffffffff),
3531                         .type_of_service = 0xff,
3532                         .next_proto_id = 0xff,
3533                 },
3534         };
3535         void *headers_m;
3536         void *headers_v;
3537         char *l24_m;
3538         char *l24_v;
3539         uint8_t tos;
3540
3541         if (inner) {
3542                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3543                                          inner_headers);
3544                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3545         } else {
3546                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3547                                          outer_headers);
3548                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3549         }
3550         if (group == 0)
3551                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3552         else
3553                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
3554         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
3555         if (!ipv4_v)
3556                 return;
3557         if (!ipv4_m)
3558                 ipv4_m = &nic_mask;
3559         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3560                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3561         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3562                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3563         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
3564         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
3565         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3566                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
3567         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3568                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
3569         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
3570         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
3571         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
3572         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
3573                  ipv4_m->hdr.type_of_service);
3574         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
3575         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
3576                  ipv4_m->hdr.type_of_service >> 2);
3577         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
3578         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3579                  ipv4_m->hdr.next_proto_id);
3580         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3581                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3582 }
3583
3584 /**
3585  * Add IPV6 item to matcher and to the value.
3586  *
3587  * @param[in, out] matcher
3588  *   Flow matcher.
3589  * @param[in, out] key
3590  *   Flow matcher value.
3591  * @param[in] item
3592  *   Flow pattern to translate.
3593  * @param[in] inner
3594  *   Item is inner pattern.
3595  * @param[in] group
3596  *   The group to insert the rule.
3597  */
3598 static void
3599 flow_dv_translate_item_ipv6(void *matcher, void *key,
3600                             const struct rte_flow_item *item,
3601                             int inner, uint32_t group)
3602 {
3603         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
3604         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
3605         const struct rte_flow_item_ipv6 nic_mask = {
3606                 .hdr = {
3607                         .src_addr =
3608                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
3609                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
3610                         .dst_addr =
3611                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
3612                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
3613                         .vtc_flow = RTE_BE32(0xffffffff),
3614                         .proto = 0xff,
3615                         .hop_limits = 0xff,
3616                 },
3617         };
3618         void *headers_m;
3619         void *headers_v;
3620         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3621         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3622         char *l24_m;
3623         char *l24_v;
3624         uint32_t vtc_m;
3625         uint32_t vtc_v;
3626         int i;
3627         int size;
3628
3629         if (inner) {
3630                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3631                                          inner_headers);
3632                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3633         } else {
3634                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3635                                          outer_headers);
3636                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3637         }
3638         if (group == 0)
3639                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3640         else
3641                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
3642         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
3643         if (!ipv6_v)
3644                 return;
3645         if (!ipv6_m)
3646                 ipv6_m = &nic_mask;
3647         size = sizeof(ipv6_m->hdr.dst_addr);
3648         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3649                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3650         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3651                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3652         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
3653         for (i = 0; i < size; ++i)
3654                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
3655         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3656                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
3657         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3658                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
3659         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
3660         for (i = 0; i < size; ++i)
3661                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
3662         /* TOS. */
3663         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
3664         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
3665         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
3666         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
3667         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
3668         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
3669         /* Label. */
3670         if (inner) {
3671                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
3672                          vtc_m);
3673                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
3674                          vtc_v);
3675         } else {
3676                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
3677                          vtc_m);
3678                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
3679                          vtc_v);
3680         }
3681         /* Protocol. */
3682         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3683                  ipv6_m->hdr.proto);
3684         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3685                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
3686 }
3687
3688 /**
3689  * Add TCP item to matcher and to the value.
3690  *
3691  * @param[in, out] matcher
3692  *   Flow matcher.
3693  * @param[in, out] key
3694  *   Flow matcher value.
3695  * @param[in] item
3696  *   Flow pattern to translate.
3697  * @param[in] inner
3698  *   Item is inner pattern.
3699  */
3700 static void
3701 flow_dv_translate_item_tcp(void *matcher, void *key,
3702                            const struct rte_flow_item *item,
3703                            int inner)
3704 {
3705         const struct rte_flow_item_tcp *tcp_m = item->mask;
3706         const struct rte_flow_item_tcp *tcp_v = item->spec;
3707         void *headers_m;
3708         void *headers_v;
3709
3710         if (inner) {
3711                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3712                                          inner_headers);
3713                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3714         } else {
3715                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3716                                          outer_headers);
3717                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3718         }
3719         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3720         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
3721         if (!tcp_v)
3722                 return;
3723         if (!tcp_m)
3724                 tcp_m = &rte_flow_item_tcp_mask;
3725         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
3726                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
3727         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
3728                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
3729         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
3730                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
3731         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
3732                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
3733         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
3734                  tcp_m->hdr.tcp_flags);
3735         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
3736                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
3737 }
3738
3739 /**
3740  * Add UDP item to matcher and to the value.
3741  *
3742  * @param[in, out] matcher
3743  *   Flow matcher.
3744  * @param[in, out] key
3745  *   Flow matcher value.
3746  * @param[in] item
3747  *   Flow pattern to translate.
3748  * @param[in] inner
3749  *   Item is inner pattern.
3750  */
3751 static void
3752 flow_dv_translate_item_udp(void *matcher, void *key,
3753                            const struct rte_flow_item *item,
3754                            int inner)
3755 {
3756         const struct rte_flow_item_udp *udp_m = item->mask;
3757         const struct rte_flow_item_udp *udp_v = item->spec;
3758         void *headers_m;
3759         void *headers_v;
3760
3761         if (inner) {
3762                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3763                                          inner_headers);
3764                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3765         } else {
3766                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3767                                          outer_headers);
3768                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3769         }
3770         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3771         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
3772         if (!udp_v)
3773                 return;
3774         if (!udp_m)
3775                 udp_m = &rte_flow_item_udp_mask;
3776         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
3777                  rte_be_to_cpu_16(udp_m->hdr.src_port));
3778         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
3779                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
3780         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
3781                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
3782         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
3783                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
3784 }
3785
3786 /**
3787  * Add GRE optional Key item to matcher and to the value.
3788  *
3789  * @param[in, out] matcher
3790  *   Flow matcher.
3791  * @param[in, out] key
3792  *   Flow matcher value.
3793  * @param[in] item
3794  *   Flow pattern to translate.
3795  * @param[in] inner
3796  *   Item is inner pattern.
3797  */
3798 static void
3799 flow_dv_translate_item_gre_key(void *matcher, void *key,
3800                                    const struct rte_flow_item *item)
3801 {
3802         const rte_be32_t *key_m = item->mask;
3803         const rte_be32_t *key_v = item->spec;
3804         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3805         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3806         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
3807
3808         if (!key_v)
3809                 return;
3810         if (!key_m)
3811                 key_m = &gre_key_default_mask;
3812         /* GRE K bit must be on and should already be validated */
3813         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
3814         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
3815         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
3816                  rte_be_to_cpu_32(*key_m) >> 8);
3817         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
3818                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
3819         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
3820                  rte_be_to_cpu_32(*key_m) & 0xFF);
3821         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
3822                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
3823 }
3824
3825 /**
3826  * Add GRE item to matcher and to the value.
3827  *
3828  * @param[in, out] matcher
3829  *   Flow matcher.
3830  * @param[in, out] key
3831  *   Flow matcher value.
3832  * @param[in] item
3833  *   Flow pattern to translate.
3834  * @param[in] inner
3835  *   Item is inner pattern.
3836  */
3837 static void
3838 flow_dv_translate_item_gre(void *matcher, void *key,
3839                            const struct rte_flow_item *item,
3840                            int inner)
3841 {
3842         const struct rte_flow_item_gre *gre_m = item->mask;
3843         const struct rte_flow_item_gre *gre_v = item->spec;
3844         void *headers_m;
3845         void *headers_v;
3846         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3847         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3848         struct {
3849                 union {
3850                         __extension__
3851                         struct {
3852                                 uint16_t version:3;
3853                                 uint16_t rsvd0:9;
3854                                 uint16_t s_present:1;
3855                                 uint16_t k_present:1;
3856                                 uint16_t rsvd_bit1:1;
3857                                 uint16_t c_present:1;
3858                         };
3859                         uint16_t value;
3860                 };
3861         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
3862
3863         if (inner) {
3864                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3865                                          inner_headers);
3866                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3867         } else {
3868                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3869                                          outer_headers);
3870                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3871         }
3872         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3873         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
3874         if (!gre_v)
3875                 return;
3876         if (!gre_m)
3877                 gre_m = &rte_flow_item_gre_mask;
3878         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
3879                  rte_be_to_cpu_16(gre_m->protocol));
3880         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
3881                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
3882         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
3883         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
3884         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
3885                  gre_crks_rsvd0_ver_m.c_present);
3886         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
3887                  gre_crks_rsvd0_ver_v.c_present &
3888                  gre_crks_rsvd0_ver_m.c_present);
3889         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
3890                  gre_crks_rsvd0_ver_m.k_present);
3891         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
3892                  gre_crks_rsvd0_ver_v.k_present &
3893                  gre_crks_rsvd0_ver_m.k_present);
3894         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
3895                  gre_crks_rsvd0_ver_m.s_present);
3896         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
3897                  gre_crks_rsvd0_ver_v.s_present &
3898                  gre_crks_rsvd0_ver_m.s_present);
3899 }
3900
3901 /**
3902  * Add NVGRE item to matcher and to the value.
3903  *
3904  * @param[in, out] matcher
3905  *   Flow matcher.
3906  * @param[in, out] key
3907  *   Flow matcher value.
3908  * @param[in] item
3909  *   Flow pattern to translate.
3910  * @param[in] inner
3911  *   Item is inner pattern.
3912  */
3913 static void
3914 flow_dv_translate_item_nvgre(void *matcher, void *key,
3915                              const struct rte_flow_item *item,
3916                              int inner)
3917 {
3918         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
3919         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
3920         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3921         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3922         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
3923         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
3924         char *gre_key_m;
3925         char *gre_key_v;
3926         int size;
3927         int i;
3928
3929         /* For NVGRE, GRE header fields must be set with defined values. */
3930         const struct rte_flow_item_gre gre_spec = {
3931                 .c_rsvd0_ver = RTE_BE16(0x2000),
3932                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
3933         };
3934         const struct rte_flow_item_gre gre_mask = {
3935                 .c_rsvd0_ver = RTE_BE16(0xB000),
3936                 .protocol = RTE_BE16(UINT16_MAX),
3937         };
3938         const struct rte_flow_item gre_item = {
3939                 .spec = &gre_spec,
3940                 .mask = &gre_mask,
3941                 .last = NULL,
3942         };
3943         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
3944         if (!nvgre_v)
3945                 return;
3946         if (!nvgre_m)
3947                 nvgre_m = &rte_flow_item_nvgre_mask;
3948         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
3949         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
3950         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
3951         memcpy(gre_key_m, tni_flow_id_m, size);
3952         for (i = 0; i < size; ++i)
3953                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
3954 }
3955
3956 /**
3957  * Add VXLAN item to matcher and to the value.
3958  *
3959  * @param[in, out] matcher
3960  *   Flow matcher.
3961  * @param[in, out] key
3962  *   Flow matcher value.
3963  * @param[in] item
3964  *   Flow pattern to translate.
3965  * @param[in] inner
3966  *   Item is inner pattern.
3967  */
3968 static void
3969 flow_dv_translate_item_vxlan(void *matcher, void *key,
3970                              const struct rte_flow_item *item,
3971                              int inner)
3972 {
3973         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
3974         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
3975         void *headers_m;
3976         void *headers_v;
3977         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3978         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3979         char *vni_m;
3980         char *vni_v;
3981         uint16_t dport;
3982         int size;
3983         int i;
3984
3985         if (inner) {
3986                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3987                                          inner_headers);
3988                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3989         } else {
3990                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3991                                          outer_headers);
3992                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3993         }
3994         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
3995                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
3996         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
3997                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
3998                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
3999         }
4000         if (!vxlan_v)
4001                 return;
4002         if (!vxlan_m)
4003                 vxlan_m = &rte_flow_item_vxlan_mask;
4004         size = sizeof(vxlan_m->vni);
4005         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4006         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4007         memcpy(vni_m, vxlan_m->vni, size);
4008         for (i = 0; i < size; ++i)
4009                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4010 }
4011
4012 /**
4013  * Add MPLS item to matcher and to the value.
4014  *
4015  * @param[in, out] matcher
4016  *   Flow matcher.
4017  * @param[in, out] key
4018  *   Flow matcher value.
4019  * @param[in] item
4020  *   Flow pattern to translate.
4021  * @param[in] prev_layer
4022  *   The protocol layer indicated in previous item.
4023  * @param[in] inner
4024  *   Item is inner pattern.
4025  */
4026 static void
4027 flow_dv_translate_item_mpls(void *matcher, void *key,
4028                             const struct rte_flow_item *item,
4029                             uint64_t prev_layer,
4030                             int inner)
4031 {
4032         const uint32_t *in_mpls_m = item->mask;
4033         const uint32_t *in_mpls_v = item->spec;
4034         uint32_t *out_mpls_m = 0;
4035         uint32_t *out_mpls_v = 0;
4036         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4037         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4038         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4039                                      misc_parameters_2);
4040         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4041         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4042         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4043
4044         switch (prev_layer) {
4045         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4046                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4047                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4048                          MLX5_UDP_PORT_MPLS);
4049                 break;
4050         case MLX5_FLOW_LAYER_GRE:
4051                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4052                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4053                          RTE_ETHER_TYPE_MPLS);
4054                 break;
4055         default:
4056                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4057                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4058                          IPPROTO_MPLS);
4059                 break;
4060         }
4061         if (!in_mpls_v)
4062                 return;
4063         if (!in_mpls_m)
4064                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4065         switch (prev_layer) {
4066         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4067                 out_mpls_m =
4068                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4069                                                  outer_first_mpls_over_udp);
4070                 out_mpls_v =
4071                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4072                                                  outer_first_mpls_over_udp);
4073                 break;
4074         case MLX5_FLOW_LAYER_GRE:
4075                 out_mpls_m =
4076                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4077                                                  outer_first_mpls_over_gre);
4078                 out_mpls_v =
4079                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4080                                                  outer_first_mpls_over_gre);
4081                 break;
4082         default:
4083                 /* Inner MPLS not over GRE is not supported. */
4084                 if (!inner) {
4085                         out_mpls_m =
4086                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4087                                                          misc2_m,
4088                                                          outer_first_mpls);
4089                         out_mpls_v =
4090                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4091                                                          misc2_v,
4092                                                          outer_first_mpls);
4093                 }
4094                 break;
4095         }
4096         if (out_mpls_m && out_mpls_v) {
4097                 *out_mpls_m = *in_mpls_m;
4098                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4099         }
4100 }
4101
4102 /**
4103  * Add META item to matcher
4104  *
4105  * @param[in, out] matcher
4106  *   Flow matcher.
4107  * @param[in, out] key
4108  *   Flow matcher value.
4109  * @param[in] item
4110  *   Flow pattern to translate.
4111  * @param[in] inner
4112  *   Item is inner pattern.
4113  */
4114 static void
4115 flow_dv_translate_item_meta(void *matcher, void *key,
4116                             const struct rte_flow_item *item)
4117 {
4118         const struct rte_flow_item_meta *meta_m;
4119         const struct rte_flow_item_meta *meta_v;
4120         void *misc2_m =
4121                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4122         void *misc2_v =
4123                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4124
4125         meta_m = (const void *)item->mask;
4126         if (!meta_m)
4127                 meta_m = &rte_flow_item_meta_mask;
4128         meta_v = (const void *)item->spec;
4129         if (meta_v) {
4130                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
4131                          rte_be_to_cpu_32(meta_m->data));
4132                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
4133                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
4134         }
4135 }
4136
4137 /**
4138  * Add source vport match to the specified matcher.
4139  *
4140  * @param[in, out] matcher
4141  *   Flow matcher.
4142  * @param[in, out] key
4143  *   Flow matcher value.
4144  * @param[in] port
4145  *   Source vport value to match
4146  * @param[in] mask
4147  *   Mask
4148  */
4149 static void
4150 flow_dv_translate_item_source_vport(void *matcher, void *key,
4151                                     int16_t port, uint16_t mask)
4152 {
4153         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4154         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4155
4156         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
4157         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
4158 }
4159
4160 /**
4161  * Translate port-id item to eswitch match on  port-id.
4162  *
4163  * @param[in] dev
4164  *   The devich to configure through.
4165  * @param[in, out] matcher
4166  *   Flow matcher.
4167  * @param[in, out] key
4168  *   Flow matcher value.
4169  * @param[in] item
4170  *   Flow pattern to translate.
4171  *
4172  * @return
4173  *   0 on success, a negative errno value otherwise.
4174  */
4175 static int
4176 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
4177                                void *key, const struct rte_flow_item *item)
4178 {
4179         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
4180         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
4181         uint16_t mask, val, id;
4182         int ret;
4183
4184         mask = pid_m ? pid_m->id : 0xffff;
4185         id = pid_v ? pid_v->id : dev->data->port_id;
4186         ret = mlx5_port_to_eswitch_info(id, NULL, &val);
4187         if (ret)
4188                 return ret;
4189         flow_dv_translate_item_source_vport(matcher, key, val, mask);
4190         return 0;
4191 }
4192
4193 /**
4194  * Add ICMP6 item to matcher and to the value.
4195  *
4196  * @param[in, out] matcher
4197  *   Flow matcher.
4198  * @param[in, out] key
4199  *   Flow matcher value.
4200  * @param[in] item
4201  *   Flow pattern to translate.
4202  * @param[in] inner
4203  *   Item is inner pattern.
4204  */
4205 static void
4206 flow_dv_translate_item_icmp6(void *matcher, void *key,
4207                               const struct rte_flow_item *item,
4208                               int inner)
4209 {
4210         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
4211         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
4212         void *headers_m;
4213         void *headers_v;
4214         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4215                                      misc_parameters_3);
4216         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4217         if (inner) {
4218                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4219                                          inner_headers);
4220                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4221         } else {
4222                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4223                                          outer_headers);
4224                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4225         }
4226         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4227         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
4228         if (!icmp6_v)
4229                 return;
4230         if (!icmp6_m)
4231                 icmp6_m = &rte_flow_item_icmp6_mask;
4232         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
4233         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
4234                  icmp6_v->type & icmp6_m->type);
4235         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
4236         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
4237                  icmp6_v->code & icmp6_m->code);
4238 }
4239
4240 /**
4241  * Add ICMP item to matcher and to the value.
4242  *
4243  * @param[in, out] matcher
4244  *   Flow matcher.
4245  * @param[in, out] key
4246  *   Flow matcher value.
4247  * @param[in] item
4248  *   Flow pattern to translate.
4249  * @param[in] inner
4250  *   Item is inner pattern.
4251  */
4252 static void
4253 flow_dv_translate_item_icmp(void *matcher, void *key,
4254                             const struct rte_flow_item *item,
4255                             int inner)
4256 {
4257         const struct rte_flow_item_icmp *icmp_m = item->mask;
4258         const struct rte_flow_item_icmp *icmp_v = item->spec;
4259         void *headers_m;
4260         void *headers_v;
4261         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4262                                      misc_parameters_3);
4263         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4264         if (inner) {
4265                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4266                                          inner_headers);
4267                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4268         } else {
4269                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4270                                          outer_headers);
4271                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4272         }
4273         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4274         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
4275         if (!icmp_v)
4276                 return;
4277         if (!icmp_m)
4278                 icmp_m = &rte_flow_item_icmp_mask;
4279         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
4280                  icmp_m->hdr.icmp_type);
4281         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
4282                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
4283         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
4284                  icmp_m->hdr.icmp_code);
4285         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
4286                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
4287 }
4288
4289 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
4290
4291 #define HEADER_IS_ZERO(match_criteria, headers)                              \
4292         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
4293                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
4294
4295 /**
4296  * Calculate flow matcher enable bitmap.
4297  *
4298  * @param match_criteria
4299  *   Pointer to flow matcher criteria.
4300  *
4301  * @return
4302  *   Bitmap of enabled fields.
4303  */
4304 static uint8_t
4305 flow_dv_matcher_enable(uint32_t *match_criteria)
4306 {
4307         uint8_t match_criteria_enable;
4308
4309         match_criteria_enable =
4310                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
4311                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
4312         match_criteria_enable |=
4313                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
4314                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
4315         match_criteria_enable |=
4316                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
4317                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
4318         match_criteria_enable |=
4319                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
4320                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
4321 #ifdef HAVE_MLX5DV_DR
4322         match_criteria_enable |=
4323                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
4324                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
4325 #endif
4326         return match_criteria_enable;
4327 }
4328
4329
4330 /**
4331  * Get a flow table.
4332  *
4333  * @param dev[in, out]
4334  *   Pointer to rte_eth_dev structure.
4335  * @param[in] table_id
4336  *   Table id to use.
4337  * @param[in] egress
4338  *   Direction of the table.
4339  * @param[in] transfer
4340  *   E-Switch or NIC flow.
4341  * @param[out] error
4342  *   pointer to error structure.
4343  *
4344  * @return
4345  *   Returns tables resource based on the index, NULL in case of failed.
4346  */
4347 static struct mlx5_flow_tbl_resource *
4348 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
4349                          uint32_t table_id, uint8_t egress,
4350                          uint8_t transfer,
4351                          struct rte_flow_error *error)
4352 {
4353         struct mlx5_priv *priv = dev->data->dev_private;
4354         struct mlx5_ibv_shared *sh = priv->sh;
4355         struct mlx5_flow_tbl_resource *tbl;
4356
4357 #ifdef HAVE_MLX5DV_DR
4358         if (transfer) {
4359                 tbl = &sh->fdb_tbl[table_id];
4360                 if (!tbl->obj)
4361                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4362                                 (sh->fdb_domain, table_id);
4363         } else if (egress) {
4364                 tbl = &sh->tx_tbl[table_id];
4365                 if (!tbl->obj)
4366                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4367                                 (sh->tx_domain, table_id);
4368         } else {
4369                 tbl = &sh->rx_tbl[table_id];
4370                 if (!tbl->obj)
4371                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4372                                 (sh->rx_domain, table_id);
4373         }
4374         if (!tbl->obj) {
4375                 rte_flow_error_set(error, ENOMEM,
4376                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4377                                    NULL, "cannot create table");
4378                 return NULL;
4379         }
4380         rte_atomic32_inc(&tbl->refcnt);
4381         return tbl;
4382 #else
4383         (void)error;
4384         (void)tbl;
4385         if (transfer)
4386                 return &sh->fdb_tbl[table_id];
4387         else if (egress)
4388                 return &sh->tx_tbl[table_id];
4389         else
4390                 return &sh->rx_tbl[table_id];
4391 #endif
4392 }
4393
4394 /**
4395  * Release a flow table.
4396  *
4397  * @param[in] tbl
4398  *   Table resource to be released.
4399  *
4400  * @return
4401  *   Returns 0 if table was released, else return 1;
4402  */
4403 static int
4404 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
4405 {
4406         if (!tbl)
4407                 return 0;
4408         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
4409                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
4410                 tbl->obj = NULL;
4411                 return 0;
4412         }
4413         return 1;
4414 }
4415
4416 /**
4417  * Register the flow matcher.
4418  *
4419  * @param dev[in, out]
4420  *   Pointer to rte_eth_dev structure.
4421  * @param[in, out] matcher
4422  *   Pointer to flow matcher.
4423  * @parm[in, out] dev_flow
4424  *   Pointer to the dev_flow.
4425  * @param[out] error
4426  *   pointer to error structure.
4427  *
4428  * @return
4429  *   0 on success otherwise -errno and errno is set.
4430  */
4431 static int
4432 flow_dv_matcher_register(struct rte_eth_dev *dev,
4433                          struct mlx5_flow_dv_matcher *matcher,
4434                          struct mlx5_flow *dev_flow,
4435                          struct rte_flow_error *error)
4436 {
4437         struct mlx5_priv *priv = dev->data->dev_private;
4438         struct mlx5_ibv_shared *sh = priv->sh;
4439         struct mlx5_flow_dv_matcher *cache_matcher;
4440         struct mlx5dv_flow_matcher_attr dv_attr = {
4441                 .type = IBV_FLOW_ATTR_NORMAL,
4442                 .match_mask = (void *)&matcher->mask,
4443         };
4444         struct mlx5_flow_tbl_resource *tbl = NULL;
4445
4446         /* Lookup from cache. */
4447         LIST_FOREACH(cache_matcher, &sh->matchers, next) {
4448                 if (matcher->crc == cache_matcher->crc &&
4449                     matcher->priority == cache_matcher->priority &&
4450                     matcher->egress == cache_matcher->egress &&
4451                     matcher->group == cache_matcher->group &&
4452                     matcher->transfer == cache_matcher->transfer &&
4453                     !memcmp((const void *)matcher->mask.buf,
4454                             (const void *)cache_matcher->mask.buf,
4455                             cache_matcher->mask.size)) {
4456                         DRV_LOG(DEBUG,
4457                                 "priority %hd use %s matcher %p: refcnt %d++",
4458                                 cache_matcher->priority,
4459                                 cache_matcher->egress ? "tx" : "rx",
4460                                 (void *)cache_matcher,
4461                                 rte_atomic32_read(&cache_matcher->refcnt));
4462                         rte_atomic32_inc(&cache_matcher->refcnt);
4463                         dev_flow->dv.matcher = cache_matcher;
4464                         return 0;
4465                 }
4466         }
4467         /* Register new matcher. */
4468         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
4469         if (!cache_matcher)
4470                 return rte_flow_error_set(error, ENOMEM,
4471                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4472                                           "cannot allocate matcher memory");
4473         tbl = flow_dv_tbl_resource_get(dev, matcher->group * MLX5_GROUP_FACTOR,
4474                                        matcher->egress, matcher->transfer,
4475                                        error);
4476         if (!tbl) {
4477                 rte_free(cache_matcher);
4478                 return rte_flow_error_set(error, ENOMEM,
4479                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4480                                           NULL, "cannot create table");
4481         }
4482         *cache_matcher = *matcher;
4483         dv_attr.match_criteria_enable =
4484                 flow_dv_matcher_enable(cache_matcher->mask.buf);
4485         dv_attr.priority = matcher->priority;
4486         if (matcher->egress)
4487                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
4488         cache_matcher->matcher_object =
4489                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
4490         if (!cache_matcher->matcher_object) {
4491                 rte_free(cache_matcher);
4492 #ifdef HAVE_MLX5DV_DR
4493                 flow_dv_tbl_resource_release(tbl);
4494 #endif
4495                 return rte_flow_error_set(error, ENOMEM,
4496                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4497                                           NULL, "cannot create matcher");
4498         }
4499         rte_atomic32_inc(&cache_matcher->refcnt);
4500         LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
4501         dev_flow->dv.matcher = cache_matcher;
4502         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
4503                 cache_matcher->priority,
4504                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
4505                 rte_atomic32_read(&cache_matcher->refcnt));
4506         rte_atomic32_inc(&tbl->refcnt);
4507         return 0;
4508 }
4509
4510 /**
4511  * Find existing tag resource or create and register a new one.
4512  *
4513  * @param dev[in, out]
4514  *   Pointer to rte_eth_dev structure.
4515  * @param[in, out] resource
4516  *   Pointer to tag resource.
4517  * @parm[in, out] dev_flow
4518  *   Pointer to the dev_flow.
4519  * @param[out] error
4520  *   pointer to error structure.
4521  *
4522  * @return
4523  *   0 on success otherwise -errno and errno is set.
4524  */
4525 static int
4526 flow_dv_tag_resource_register
4527                         (struct rte_eth_dev *dev,
4528                          struct mlx5_flow_dv_tag_resource *resource,
4529                          struct mlx5_flow *dev_flow,
4530                          struct rte_flow_error *error)
4531 {
4532         struct mlx5_priv *priv = dev->data->dev_private;
4533         struct mlx5_ibv_shared *sh = priv->sh;
4534         struct mlx5_flow_dv_tag_resource *cache_resource;
4535
4536         /* Lookup a matching resource from cache. */
4537         LIST_FOREACH(cache_resource, &sh->tags, next) {
4538                 if (resource->tag == cache_resource->tag) {
4539                         DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
4540                                 (void *)cache_resource,
4541                                 rte_atomic32_read(&cache_resource->refcnt));
4542                         rte_atomic32_inc(&cache_resource->refcnt);
4543                         dev_flow->flow->tag_resource = cache_resource;
4544                         return 0;
4545                 }
4546         }
4547         /* Register new  resource. */
4548         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
4549         if (!cache_resource)
4550                 return rte_flow_error_set(error, ENOMEM,
4551                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4552                                           "cannot allocate resource memory");
4553         *cache_resource = *resource;
4554         cache_resource->action = mlx5_glue->dv_create_flow_action_tag
4555                 (resource->tag);
4556         if (!cache_resource->action) {
4557                 rte_free(cache_resource);
4558                 return rte_flow_error_set(error, ENOMEM,
4559                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4560                                           NULL, "cannot create action");
4561         }
4562         rte_atomic32_init(&cache_resource->refcnt);
4563         rte_atomic32_inc(&cache_resource->refcnt);
4564         LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
4565         dev_flow->flow->tag_resource = cache_resource;
4566         DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
4567                 (void *)cache_resource,
4568                 rte_atomic32_read(&cache_resource->refcnt));
4569         return 0;
4570 }
4571
4572 /**
4573  * Release the tag.
4574  *
4575  * @param dev
4576  *   Pointer to Ethernet device.
4577  * @param flow
4578  *   Pointer to mlx5_flow.
4579  *
4580  * @return
4581  *   1 while a reference on it exists, 0 when freed.
4582  */
4583 static int
4584 flow_dv_tag_release(struct rte_eth_dev *dev,
4585                     struct mlx5_flow_dv_tag_resource *tag)
4586 {
4587         assert(tag);
4588         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
4589                 dev->data->port_id, (void *)tag,
4590                 rte_atomic32_read(&tag->refcnt));
4591         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
4592                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
4593                 LIST_REMOVE(tag, next);
4594                 DRV_LOG(DEBUG, "port %u tag %p: removed",
4595                         dev->data->port_id, (void *)tag);
4596                 rte_free(tag);
4597                 return 0;
4598         }
4599         return 1;
4600 }
4601
4602 /**
4603  * Translate port ID action to vport.
4604  *
4605  * @param[in] dev
4606  *   Pointer to rte_eth_dev structure.
4607  * @param[in] action
4608  *   Pointer to the port ID action.
4609  * @param[out] dst_port_id
4610  *   The target port ID.
4611  * @param[out] error
4612  *   Pointer to the error structure.
4613  *
4614  * @return
4615  *   0 on success, a negative errno value otherwise and rte_errno is set.
4616  */
4617 static int
4618 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
4619                                  const struct rte_flow_action *action,
4620                                  uint32_t *dst_port_id,
4621                                  struct rte_flow_error *error)
4622 {
4623         uint32_t port;
4624         uint16_t port_id;
4625         int ret;
4626         const struct rte_flow_action_port_id *conf =
4627                         (const struct rte_flow_action_port_id *)action->conf;
4628
4629         port = conf->original ? dev->data->port_id : conf->id;
4630         ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
4631         if (ret)
4632                 return rte_flow_error_set(error, -ret,
4633                                           RTE_FLOW_ERROR_TYPE_ACTION,
4634                                           NULL,
4635                                           "No eswitch info was found for port");
4636         *dst_port_id = port_id;
4637         return 0;
4638 }
4639
4640 /**
4641  * Fill the flow with DV spec.
4642  *
4643  * @param[in] dev
4644  *   Pointer to rte_eth_dev structure.
4645  * @param[in, out] dev_flow
4646  *   Pointer to the sub flow.
4647  * @param[in] attr
4648  *   Pointer to the flow attributes.
4649  * @param[in] items
4650  *   Pointer to the list of items.
4651  * @param[in] actions
4652  *   Pointer to the list of actions.
4653  * @param[out] error
4654  *   Pointer to the error structure.
4655  *
4656  * @return
4657  *   0 on success, a negative errno value otherwise and rte_errno is set.
4658  */
4659 static int
4660 flow_dv_translate(struct rte_eth_dev *dev,
4661                   struct mlx5_flow *dev_flow,
4662                   const struct rte_flow_attr *attr,
4663                   const struct rte_flow_item items[],
4664                   const struct rte_flow_action actions[],
4665                   struct rte_flow_error *error)
4666 {
4667         struct mlx5_priv *priv = dev->data->dev_private;
4668         struct rte_flow *flow = dev_flow->flow;
4669         uint64_t item_flags = 0;
4670         uint64_t last_item = 0;
4671         uint64_t action_flags = 0;
4672         uint64_t priority = attr->priority;
4673         struct mlx5_flow_dv_matcher matcher = {
4674                 .mask = {
4675                         .size = sizeof(matcher.mask.buf),
4676                 },
4677         };
4678         int actions_n = 0;
4679         bool actions_end = false;
4680         struct mlx5_flow_dv_modify_hdr_resource res = {
4681                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4682                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
4683         };
4684         union flow_dv_attr flow_attr = { .attr = 0 };
4685         struct mlx5_flow_dv_tag_resource tag_resource;
4686         uint32_t modify_action_position = UINT32_MAX;
4687         void *match_mask = matcher.mask.buf;
4688         void *match_value = dev_flow->dv.value.buf;
4689
4690         flow->group = attr->group;
4691         if (attr->transfer)
4692                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4693         if (priority == MLX5_FLOW_PRIO_RSVD)
4694                 priority = priv->config.flow_prio - 1;
4695         for (; !actions_end ; actions++) {
4696                 const struct rte_flow_action_queue *queue;
4697                 const struct rte_flow_action_rss *rss;
4698                 const struct rte_flow_action *action = actions;
4699                 const struct rte_flow_action_count *count = action->conf;
4700                 const uint8_t *rss_key;
4701                 const struct rte_flow_action_jump *jump_data;
4702                 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
4703                 struct mlx5_flow_tbl_resource *tbl;
4704                 uint32_t port_id = 0;
4705                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
4706
4707                 switch (actions->type) {
4708                 case RTE_FLOW_ACTION_TYPE_VOID:
4709                         break;
4710                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4711                         if (flow_dv_translate_action_port_id(dev, action,
4712                                                              &port_id, error))
4713                                 return -rte_errno;
4714                         port_id_resource.port_id = port_id;
4715                         if (flow_dv_port_id_action_resource_register
4716                             (dev, &port_id_resource, dev_flow, error))
4717                                 return -rte_errno;
4718                         dev_flow->dv.actions[actions_n++] =
4719                                 dev_flow->dv.port_id_action->action;
4720                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4721                         break;
4722                 case RTE_FLOW_ACTION_TYPE_FLAG:
4723                         tag_resource.tag =
4724                                 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
4725                         if (!flow->tag_resource)
4726                                 if (flow_dv_tag_resource_register
4727                                     (dev, &tag_resource, dev_flow, error))
4728                                         return errno;
4729                         dev_flow->dv.actions[actions_n++] =
4730                                 flow->tag_resource->action;
4731                         action_flags |= MLX5_FLOW_ACTION_FLAG;
4732                         break;
4733                 case RTE_FLOW_ACTION_TYPE_MARK:
4734                         tag_resource.tag = mlx5_flow_mark_set
4735                               (((const struct rte_flow_action_mark *)
4736                                (actions->conf))->id);
4737                         if (!flow->tag_resource)
4738                                 if (flow_dv_tag_resource_register
4739                                     (dev, &tag_resource, dev_flow, error))
4740                                         return errno;
4741                         dev_flow->dv.actions[actions_n++] =
4742                                 flow->tag_resource->action;
4743                         action_flags |= MLX5_FLOW_ACTION_MARK;
4744                         break;
4745                 case RTE_FLOW_ACTION_TYPE_DROP:
4746                         action_flags |= MLX5_FLOW_ACTION_DROP;
4747                         break;
4748                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4749                         queue = actions->conf;
4750                         flow->rss.queue_num = 1;
4751                         (*flow->queue)[0] = queue->index;
4752                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
4753                         break;
4754                 case RTE_FLOW_ACTION_TYPE_RSS:
4755                         rss = actions->conf;
4756                         if (flow->queue)
4757                                 memcpy((*flow->queue), rss->queue,
4758                                        rss->queue_num * sizeof(uint16_t));
4759                         flow->rss.queue_num = rss->queue_num;
4760                         /* NULL RSS key indicates default RSS key. */
4761                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
4762                         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
4763                         /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
4764                         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
4765                         flow->rss.level = rss->level;
4766                         action_flags |= MLX5_FLOW_ACTION_RSS;
4767                         break;
4768                 case RTE_FLOW_ACTION_TYPE_COUNT:
4769                         if (!priv->config.devx) {
4770                                 rte_errno = ENOTSUP;
4771                                 goto cnt_err;
4772                         }
4773                         flow->counter = flow_dv_counter_alloc(dev,
4774                                                               count->shared,
4775                                                               count->id,
4776                                                               attr->group);
4777                         if (flow->counter == NULL)
4778                                 goto cnt_err;
4779                         dev_flow->dv.actions[actions_n++] =
4780                                 flow->counter->action;
4781                         action_flags |= MLX5_FLOW_ACTION_COUNT;
4782                         break;
4783 cnt_err:
4784                         if (rte_errno == ENOTSUP)
4785                                 return rte_flow_error_set
4786                                               (error, ENOTSUP,
4787                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4788                                                NULL,
4789                                                "count action not supported");
4790                         else
4791                                 return rte_flow_error_set
4792                                                 (error, rte_errno,
4793                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4794                                                  action,
4795                                                  "cannot create counter"
4796                                                   " object.");
4797                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4798                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4799                         if (flow_dv_create_action_l2_encap(dev, actions,
4800                                                            dev_flow,
4801                                                            attr->transfer,
4802                                                            error))
4803                                 return -rte_errno;
4804                         dev_flow->dv.actions[actions_n++] =
4805                                 dev_flow->dv.encap_decap->verbs_action;
4806                         action_flags |= actions->type ==
4807                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
4808                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
4809                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
4810                         break;
4811                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4812                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4813                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
4814                                                            attr->transfer,
4815                                                            error))
4816                                 return -rte_errno;
4817                         dev_flow->dv.actions[actions_n++] =
4818                                 dev_flow->dv.encap_decap->verbs_action;
4819                         action_flags |= actions->type ==
4820                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
4821                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
4822                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
4823                         break;
4824                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4825                         /* Handle encap with preceding decap. */
4826                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
4827                                 if (flow_dv_create_action_raw_encap
4828                                         (dev, actions, dev_flow, attr, error))
4829                                         return -rte_errno;
4830                                 dev_flow->dv.actions[actions_n++] =
4831                                         dev_flow->dv.encap_decap->verbs_action;
4832                         } else {
4833                                 /* Handle encap without preceding decap. */
4834                                 if (flow_dv_create_action_l2_encap
4835                                     (dev, actions, dev_flow, attr->transfer,
4836                                      error))
4837                                         return -rte_errno;
4838                                 dev_flow->dv.actions[actions_n++] =
4839                                         dev_flow->dv.encap_decap->verbs_action;
4840                         }
4841                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
4842                         break;
4843                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4844                         /* Check if this decap is followed by encap. */
4845                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
4846                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
4847                                action++) {
4848                         }
4849                         /* Handle decap only if it isn't followed by encap. */
4850                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4851                                 if (flow_dv_create_action_l2_decap
4852                                     (dev, dev_flow, attr->transfer, error))
4853                                         return -rte_errno;
4854                                 dev_flow->dv.actions[actions_n++] =
4855                                         dev_flow->dv.encap_decap->verbs_action;
4856                         }
4857                         /* If decap is followed by encap, handle it at encap. */
4858                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
4859                         break;
4860                 case RTE_FLOW_ACTION_TYPE_JUMP:
4861                         jump_data = action->conf;
4862                         tbl = flow_dv_tbl_resource_get(dev, jump_data->group *
4863                                                        MLX5_GROUP_FACTOR,
4864                                                        attr->egress,
4865                                                        attr->transfer, error);
4866                         if (!tbl)
4867                                 return rte_flow_error_set
4868                                                 (error, errno,
4869                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4870                                                  NULL,
4871                                                  "cannot create jump action.");
4872                         jump_tbl_resource.tbl = tbl;
4873                         if (flow_dv_jump_tbl_resource_register
4874                             (dev, &jump_tbl_resource, dev_flow, error)) {
4875                                 flow_dv_tbl_resource_release(tbl);
4876                                 return rte_flow_error_set
4877                                                 (error, errno,
4878                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4879                                                  NULL,
4880                                                  "cannot create jump action.");
4881                         }
4882                         dev_flow->dv.actions[actions_n++] =
4883                                 dev_flow->dv.jump->action;
4884                         action_flags |= MLX5_FLOW_ACTION_JUMP;
4885                         break;
4886                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4887                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4888                         if (flow_dv_convert_action_modify_mac(&res, actions,
4889                                                               error))
4890                                 return -rte_errno;
4891                         action_flags |= actions->type ==
4892                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
4893                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
4894                                         MLX5_FLOW_ACTION_SET_MAC_DST;
4895                         break;
4896                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4897                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4898                         if (flow_dv_convert_action_modify_ipv4(&res, actions,
4899                                                                error))
4900                                 return -rte_errno;
4901                         action_flags |= actions->type ==
4902                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
4903                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
4904                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
4905                         break;
4906                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4907                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4908                         if (flow_dv_convert_action_modify_ipv6(&res, actions,
4909                                                                error))
4910                                 return -rte_errno;
4911                         action_flags |= actions->type ==
4912                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
4913                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
4914                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
4915                         break;
4916                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4917                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4918                         if (flow_dv_convert_action_modify_tp(&res, actions,
4919                                                              items, &flow_attr,
4920                                                              error))
4921                                 return -rte_errno;
4922                         action_flags |= actions->type ==
4923                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
4924                                         MLX5_FLOW_ACTION_SET_TP_SRC :
4925                                         MLX5_FLOW_ACTION_SET_TP_DST;
4926                         break;
4927                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4928                         if (flow_dv_convert_action_modify_dec_ttl(&res, items,
4929                                                                   &flow_attr,
4930                                                                   error))
4931                                 return -rte_errno;
4932                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
4933                         break;
4934                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
4935                         if (flow_dv_convert_action_modify_ttl(&res, actions,
4936                                                              items, &flow_attr,
4937                                                              error))
4938                                 return -rte_errno;
4939                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
4940                         break;
4941                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4942                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4943                         if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
4944                                                                   error))
4945                                 return -rte_errno;
4946                         action_flags |= actions->type ==
4947                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
4948                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
4949                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
4950                         break;
4951
4952                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4953                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4954                         if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
4955                                                                   error))
4956                                 return -rte_errno;
4957                         action_flags |= actions->type ==
4958                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
4959                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
4960                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
4961                         break;
4962                 case RTE_FLOW_ACTION_TYPE_END:
4963                         actions_end = true;
4964                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
4965                                 /* create modify action if needed. */
4966                                 if (flow_dv_modify_hdr_resource_register
4967                                                                 (dev, &res,
4968                                                                  dev_flow,
4969                                                                  error))
4970                                         return -rte_errno;
4971                                 dev_flow->dv.actions[modify_action_position] =
4972                                         dev_flow->dv.modify_hdr->verbs_action;
4973                         }
4974                         break;
4975                 default:
4976                         break;
4977                 }
4978                 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
4979                     modify_action_position == UINT32_MAX)
4980                         modify_action_position = actions_n++;
4981         }
4982         dev_flow->dv.actions_n = actions_n;
4983         flow->actions = action_flags;
4984         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4985                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4986
4987                 switch (items->type) {
4988                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4989                         flow_dv_translate_item_port_id(dev, match_mask,
4990                                                        match_value, items);
4991                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4992                         break;
4993                 case RTE_FLOW_ITEM_TYPE_ETH:
4994                         flow_dv_translate_item_eth(match_mask, match_value,
4995                                                    items, tunnel);
4996                         matcher.priority = MLX5_PRIORITY_MAP_L2;
4997                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4998                                              MLX5_FLOW_LAYER_OUTER_L2;
4999                         break;
5000                 case RTE_FLOW_ITEM_TYPE_VLAN:
5001                         flow_dv_translate_item_vlan(match_mask, match_value,
5002                                                     items, tunnel);
5003                         matcher.priority = MLX5_PRIORITY_MAP_L2;
5004                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
5005                                               MLX5_FLOW_LAYER_INNER_VLAN) :
5006                                              (MLX5_FLOW_LAYER_OUTER_L2 |
5007                                               MLX5_FLOW_LAYER_OUTER_VLAN);
5008                         break;
5009                 case RTE_FLOW_ITEM_TYPE_IPV4:
5010                         flow_dv_translate_item_ipv4(match_mask, match_value,
5011                                                     items, tunnel, attr->group);
5012                         matcher.priority = MLX5_PRIORITY_MAP_L3;
5013                         dev_flow->dv.hash_fields |=
5014                                 mlx5_flow_hashfields_adjust
5015                                         (dev_flow, tunnel,
5016                                          MLX5_IPV4_LAYER_TYPES,
5017                                          MLX5_IPV4_IBV_RX_HASH);
5018                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5019                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5020                         mlx5_flow_tunnel_ip_check(items, &last_item);
5021                         break;
5022                 case RTE_FLOW_ITEM_TYPE_IPV6:
5023                         flow_dv_translate_item_ipv6(match_mask, match_value,
5024                                                     items, tunnel, attr->group);
5025                         matcher.priority = MLX5_PRIORITY_MAP_L3;
5026                         dev_flow->dv.hash_fields |=
5027                                 mlx5_flow_hashfields_adjust
5028                                         (dev_flow, tunnel,
5029                                          MLX5_IPV6_LAYER_TYPES,
5030                                          MLX5_IPV6_IBV_RX_HASH);
5031                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5032                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5033                         mlx5_flow_tunnel_ip_check(items, &last_item);
5034                         break;
5035                 case RTE_FLOW_ITEM_TYPE_TCP:
5036                         flow_dv_translate_item_tcp(match_mask, match_value,
5037                                                    items, tunnel);
5038                         matcher.priority = MLX5_PRIORITY_MAP_L4;
5039                         dev_flow->dv.hash_fields |=
5040                                 mlx5_flow_hashfields_adjust
5041                                         (dev_flow, tunnel, ETH_RSS_TCP,
5042                                          IBV_RX_HASH_SRC_PORT_TCP |
5043                                          IBV_RX_HASH_DST_PORT_TCP);
5044                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5045                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5046                         break;
5047                 case RTE_FLOW_ITEM_TYPE_UDP:
5048                         flow_dv_translate_item_udp(match_mask, match_value,
5049                                                    items, tunnel);
5050                         matcher.priority = MLX5_PRIORITY_MAP_L4;
5051                         dev_flow->dv.hash_fields |=
5052                                 mlx5_flow_hashfields_adjust
5053                                         (dev_flow, tunnel, ETH_RSS_UDP,
5054                                          IBV_RX_HASH_SRC_PORT_UDP |
5055                                          IBV_RX_HASH_DST_PORT_UDP);
5056                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5057                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5058                         break;
5059                 case RTE_FLOW_ITEM_TYPE_GRE:
5060                         flow_dv_translate_item_gre(match_mask, match_value,
5061                                                    items, tunnel);
5062                         last_item = MLX5_FLOW_LAYER_GRE;
5063                         break;
5064                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5065                         flow_dv_translate_item_gre_key(match_mask,
5066                                                        match_value, items);
5067                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5068                         break;
5069                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5070                         flow_dv_translate_item_nvgre(match_mask, match_value,
5071                                                      items, tunnel);
5072                         last_item = MLX5_FLOW_LAYER_GRE;
5073                         break;
5074                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5075                         flow_dv_translate_item_vxlan(match_mask, match_value,
5076                                                      items, tunnel);
5077                         last_item = MLX5_FLOW_LAYER_VXLAN;
5078                         break;
5079                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5080                         flow_dv_translate_item_vxlan(match_mask, match_value,
5081                                                      items, tunnel);
5082                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5083                         break;
5084                 case RTE_FLOW_ITEM_TYPE_MPLS:
5085                         flow_dv_translate_item_mpls(match_mask, match_value,
5086                                                     items, last_item, tunnel);
5087                         last_item = MLX5_FLOW_LAYER_MPLS;
5088                         break;
5089                 case RTE_FLOW_ITEM_TYPE_META:
5090                         flow_dv_translate_item_meta(match_mask, match_value,
5091                                                     items);
5092                         last_item = MLX5_FLOW_ITEM_METADATA;
5093                         break;
5094                 case RTE_FLOW_ITEM_TYPE_ICMP:
5095                         flow_dv_translate_item_icmp(match_mask, match_value,
5096                                                     items, tunnel);
5097                         last_item = MLX5_FLOW_LAYER_ICMP;
5098                         break;
5099                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5100                         flow_dv_translate_item_icmp6(match_mask, match_value,
5101                                                       items, tunnel);
5102                         last_item = MLX5_FLOW_LAYER_ICMP6;
5103                         break;
5104                 default:
5105                         break;
5106                 }
5107                 item_flags |= last_item;
5108         }
5109         /*
5110          * In case of ingress traffic when E-Switch mode is enabled,
5111          * we have two cases where we need to set the source port manually.
5112          * The first one, is in case of Nic steering rule, and the second is
5113          * E-Switch rule where no port_id item was found. In both cases
5114          * the source port is set according the current port in use.
5115          */
5116         if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
5117             (priv->representor || priv->master)) {
5118                 if (flow_dv_translate_item_port_id(dev, match_mask,
5119                                                    match_value, NULL))
5120                         return -rte_errno;
5121         }
5122         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
5123                                          dev_flow->dv.value.buf));
5124         dev_flow->layers = item_flags;
5125         /* Register matcher. */
5126         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
5127                                     matcher.mask.size);
5128         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
5129                                                      matcher.priority);
5130         matcher.egress = attr->egress;
5131         matcher.group = attr->group;
5132         matcher.transfer = attr->transfer;
5133         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
5134                 return -rte_errno;
5135         return 0;
5136 }
5137
5138 /**
5139  * Apply the flow to the NIC.
5140  *
5141  * @param[in] dev
5142  *   Pointer to the Ethernet device structure.
5143  * @param[in, out] flow
5144  *   Pointer to flow structure.
5145  * @param[out] error
5146  *   Pointer to error structure.
5147  *
5148  * @return
5149  *   0 on success, a negative errno value otherwise and rte_errno is set.
5150  */
5151 static int
5152 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5153               struct rte_flow_error *error)
5154 {
5155         struct mlx5_flow_dv *dv;
5156         struct mlx5_flow *dev_flow;
5157         struct mlx5_priv *priv = dev->data->dev_private;
5158         int n;
5159         int err;
5160
5161         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5162                 dv = &dev_flow->dv;
5163                 n = dv->actions_n;
5164                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
5165                         if (flow->transfer) {
5166                                 dv->actions[n++] = priv->sh->esw_drop_action;
5167                         } else {
5168                                 dv->hrxq = mlx5_hrxq_drop_new(dev);
5169                                 if (!dv->hrxq) {
5170                                         rte_flow_error_set
5171                                                 (error, errno,
5172                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5173                                                  NULL,
5174                                                  "cannot get drop hash queue");
5175                                         goto error;
5176                                 }
5177                                 dv->actions[n++] = dv->hrxq->action;
5178                         }
5179                 } else if (flow->actions &
5180                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
5181                         struct mlx5_hrxq *hrxq;
5182
5183                         hrxq = mlx5_hrxq_get(dev, flow->key,
5184                                              MLX5_RSS_HASH_KEY_LEN,
5185                                              dv->hash_fields,
5186                                              (*flow->queue),
5187                                              flow->rss.queue_num);
5188                         if (!hrxq) {
5189                                 int lro = 0;
5190
5191                                 if (mlx5_lro_on(dev)) {
5192                                         if ((dev_flow->layers &
5193                                              MLX5_FLOW_LAYER_IPV4_LRO)
5194                                             == MLX5_FLOW_LAYER_IPV4_LRO)
5195                                                 lro = MLX5_FLOW_IPV4_LRO;
5196                                         else if ((dev_flow->layers &
5197                                                   MLX5_FLOW_LAYER_IPV6_LRO)
5198                                                  == MLX5_FLOW_LAYER_IPV6_LRO)
5199                                                 lro = MLX5_FLOW_IPV6_LRO;
5200                                 }
5201                                 hrxq = mlx5_hrxq_new
5202                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
5203                                          dv->hash_fields, (*flow->queue),
5204                                          flow->rss.queue_num,
5205                                          !!(dev_flow->layers &
5206                                             MLX5_FLOW_LAYER_TUNNEL), lro);
5207                         }
5208
5209                         if (!hrxq) {
5210                                 rte_flow_error_set
5211                                         (error, rte_errno,
5212                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5213                                          "cannot get hash queue");
5214                                 goto error;
5215                         }
5216                         dv->hrxq = hrxq;
5217                         dv->actions[n++] = dv->hrxq->action;
5218                 }
5219                 dv->flow =
5220                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
5221                                                   (void *)&dv->value, n,
5222                                                   dv->actions);
5223                 if (!dv->flow) {
5224                         rte_flow_error_set(error, errno,
5225                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5226                                            NULL,
5227                                            "hardware refuses to create flow");
5228                         goto error;
5229                 }
5230         }
5231         return 0;
5232 error:
5233         err = rte_errno; /* Save rte_errno before cleanup. */
5234         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5235                 struct mlx5_flow_dv *dv = &dev_flow->dv;
5236                 if (dv->hrxq) {
5237                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
5238                                 mlx5_hrxq_drop_release(dev);
5239                         else
5240                                 mlx5_hrxq_release(dev, dv->hrxq);
5241                         dv->hrxq = NULL;
5242                 }
5243         }
5244         rte_errno = err; /* Restore rte_errno. */
5245         return -rte_errno;
5246 }
5247
5248 /**
5249  * Release the flow matcher.
5250  *
5251  * @param dev
5252  *   Pointer to Ethernet device.
5253  * @param flow
5254  *   Pointer to mlx5_flow.
5255  *
5256  * @return
5257  *   1 while a reference on it exists, 0 when freed.
5258  */
5259 static int
5260 flow_dv_matcher_release(struct rte_eth_dev *dev,
5261                         struct mlx5_flow *flow)
5262 {
5263         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
5264         struct mlx5_priv *priv = dev->data->dev_private;
5265         struct mlx5_ibv_shared *sh = priv->sh;
5266         struct mlx5_flow_tbl_resource *tbl;
5267
5268         assert(matcher->matcher_object);
5269         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
5270                 dev->data->port_id, (void *)matcher,
5271                 rte_atomic32_read(&matcher->refcnt));
5272         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
5273                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
5274                            (matcher->matcher_object));
5275                 LIST_REMOVE(matcher, next);
5276                 if (matcher->egress)
5277                         tbl = &sh->tx_tbl[matcher->group];
5278                 else
5279                         tbl = &sh->rx_tbl[matcher->group];
5280                 flow_dv_tbl_resource_release(tbl);
5281                 rte_free(matcher);
5282                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
5283                         dev->data->port_id, (void *)matcher);
5284                 return 0;
5285         }
5286         return 1;
5287 }
5288
5289 /**
5290  * Release an encap/decap resource.
5291  *
5292  * @param flow
5293  *   Pointer to mlx5_flow.
5294  *
5295  * @return
5296  *   1 while a reference on it exists, 0 when freed.
5297  */
5298 static int
5299 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
5300 {
5301         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
5302                                                 flow->dv.encap_decap;
5303
5304         assert(cache_resource->verbs_action);
5305         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
5306                 (void *)cache_resource,
5307                 rte_atomic32_read(&cache_resource->refcnt));
5308         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5309                 claim_zero(mlx5_glue->destroy_flow_action
5310                                 (cache_resource->verbs_action));
5311                 LIST_REMOVE(cache_resource, next);
5312                 rte_free(cache_resource);
5313                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
5314                         (void *)cache_resource);
5315                 return 0;
5316         }
5317         return 1;
5318 }
5319
5320 /**
5321  * Release an jump to table action resource.
5322  *
5323  * @param flow
5324  *   Pointer to mlx5_flow.
5325  *
5326  * @return
5327  *   1 while a reference on it exists, 0 when freed.
5328  */
5329 static int
5330 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
5331 {
5332         struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
5333                                                 flow->dv.jump;
5334
5335         assert(cache_resource->action);
5336         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
5337                 (void *)cache_resource,
5338                 rte_atomic32_read(&cache_resource->refcnt));
5339         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5340                 claim_zero(mlx5_glue->destroy_flow_action
5341                                 (cache_resource->action));
5342                 LIST_REMOVE(cache_resource, next);
5343                 flow_dv_tbl_resource_release(cache_resource->tbl);
5344                 rte_free(cache_resource);
5345                 DRV_LOG(DEBUG, "jump table resource %p: removed",
5346                         (void *)cache_resource);
5347                 return 0;
5348         }
5349         return 1;
5350 }
5351
5352 /**
5353  * Release a modify-header resource.
5354  *
5355  * @param flow
5356  *   Pointer to mlx5_flow.
5357  *
5358  * @return
5359  *   1 while a reference on it exists, 0 when freed.
5360  */
5361 static int
5362 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
5363 {
5364         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
5365                                                 flow->dv.modify_hdr;
5366
5367         assert(cache_resource->verbs_action);
5368         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
5369                 (void *)cache_resource,
5370                 rte_atomic32_read(&cache_resource->refcnt));
5371         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5372                 claim_zero(mlx5_glue->destroy_flow_action
5373                                 (cache_resource->verbs_action));
5374                 LIST_REMOVE(cache_resource, next);
5375                 rte_free(cache_resource);
5376                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
5377                         (void *)cache_resource);
5378                 return 0;
5379         }
5380         return 1;
5381 }
5382
5383 /**
5384  * Release port ID action resource.
5385  *
5386  * @param flow
5387  *   Pointer to mlx5_flow.
5388  *
5389  * @return
5390  *   1 while a reference on it exists, 0 when freed.
5391  */
5392 static int
5393 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
5394 {
5395         struct mlx5_flow_dv_port_id_action_resource *cache_resource =
5396                 flow->dv.port_id_action;
5397
5398         assert(cache_resource->action);
5399         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
5400                 (void *)cache_resource,
5401                 rte_atomic32_read(&cache_resource->refcnt));
5402         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5403                 claim_zero(mlx5_glue->destroy_flow_action
5404                                 (cache_resource->action));
5405                 LIST_REMOVE(cache_resource, next);
5406                 rte_free(cache_resource);
5407                 DRV_LOG(DEBUG, "port id action resource %p: removed",
5408                         (void *)cache_resource);
5409                 return 0;
5410         }
5411         return 1;
5412 }
5413
5414 /**
5415  * Remove the flow from the NIC but keeps it in memory.
5416  *
5417  * @param[in] dev
5418  *   Pointer to Ethernet device.
5419  * @param[in, out] flow
5420  *   Pointer to flow structure.
5421  */
5422 static void
5423 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5424 {
5425         struct mlx5_flow_dv *dv;
5426         struct mlx5_flow *dev_flow;
5427
5428         if (!flow)
5429                 return;
5430         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5431                 dv = &dev_flow->dv;
5432                 if (dv->flow) {
5433                         claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
5434                         dv->flow = NULL;
5435                 }
5436                 if (dv->hrxq) {
5437                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
5438                                 mlx5_hrxq_drop_release(dev);
5439                         else
5440                                 mlx5_hrxq_release(dev, dv->hrxq);
5441                         dv->hrxq = NULL;
5442                 }
5443         }
5444 }
5445
5446 /**
5447  * Remove the flow from the NIC and the memory.
5448  *
5449  * @param[in] dev
5450  *   Pointer to the Ethernet device structure.
5451  * @param[in, out] flow
5452  *   Pointer to flow structure.
5453  */
5454 static void
5455 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5456 {
5457         struct mlx5_flow *dev_flow;
5458
5459         if (!flow)
5460                 return;
5461         flow_dv_remove(dev, flow);
5462         if (flow->counter) {
5463                 flow_dv_counter_release(dev, flow->counter);
5464                 flow->counter = NULL;
5465         }
5466         if (flow->tag_resource) {
5467                 flow_dv_tag_release(dev, flow->tag_resource);
5468                 flow->tag_resource = NULL;
5469         }
5470         while (!LIST_EMPTY(&flow->dev_flows)) {
5471                 dev_flow = LIST_FIRST(&flow->dev_flows);
5472                 LIST_REMOVE(dev_flow, next);
5473                 if (dev_flow->dv.matcher)
5474                         flow_dv_matcher_release(dev, dev_flow);
5475                 if (dev_flow->dv.encap_decap)
5476                         flow_dv_encap_decap_resource_release(dev_flow);
5477                 if (dev_flow->dv.modify_hdr)
5478                         flow_dv_modify_hdr_resource_release(dev_flow);
5479                 if (dev_flow->dv.jump)
5480                         flow_dv_jump_tbl_resource_release(dev_flow);
5481                 if (dev_flow->dv.port_id_action)
5482                         flow_dv_port_id_action_resource_release(dev_flow);
5483                 rte_free(dev_flow);
5484         }
5485 }
5486
5487 /**
5488  * Query a dv flow  rule for its statistics via devx.
5489  *
5490  * @param[in] dev
5491  *   Pointer to Ethernet device.
5492  * @param[in] flow
5493  *   Pointer to the sub flow.
5494  * @param[out] data
5495  *   data retrieved by the query.
5496  * @param[out] error
5497  *   Perform verbose error reporting if not NULL.
5498  *
5499  * @return
5500  *   0 on success, a negative errno value otherwise and rte_errno is set.
5501  */
5502 static int
5503 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
5504                     void *data, struct rte_flow_error *error)
5505 {
5506         struct mlx5_priv *priv = dev->data->dev_private;
5507         struct rte_flow_query_count *qc = data;
5508
5509         if (!priv->config.devx)
5510                 return rte_flow_error_set(error, ENOTSUP,
5511                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5512                                           NULL,
5513                                           "counters are not supported");
5514         if (flow->counter) {
5515                 uint64_t pkts, bytes;
5516                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
5517                                                &bytes);
5518
5519                 if (err)
5520                         return rte_flow_error_set(error, -err,
5521                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5522                                         NULL, "cannot read counters");
5523                 qc->hits_set = 1;
5524                 qc->bytes_set = 1;
5525                 qc->hits = pkts - flow->counter->hits;
5526                 qc->bytes = bytes - flow->counter->bytes;
5527                 if (qc->reset) {
5528                         flow->counter->hits = pkts;
5529                         flow->counter->bytes = bytes;
5530                 }
5531                 return 0;
5532         }
5533         return rte_flow_error_set(error, EINVAL,
5534                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5535                                   NULL,
5536                                   "counters are not available");
5537 }
5538
5539 /**
5540  * Query a flow.
5541  *
5542  * @see rte_flow_query()
5543  * @see rte_flow_ops
5544  */
5545 static int
5546 flow_dv_query(struct rte_eth_dev *dev,
5547               struct rte_flow *flow __rte_unused,
5548               const struct rte_flow_action *actions __rte_unused,
5549               void *data __rte_unused,
5550               struct rte_flow_error *error __rte_unused)
5551 {
5552         int ret = -EINVAL;
5553
5554         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5555                 switch (actions->type) {
5556                 case RTE_FLOW_ACTION_TYPE_VOID:
5557                         break;
5558                 case RTE_FLOW_ACTION_TYPE_COUNT:
5559                         ret = flow_dv_query_count(dev, flow, data, error);
5560                         break;
5561                 default:
5562                         return rte_flow_error_set(error, ENOTSUP,
5563                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5564                                                   actions,
5565                                                   "action not supported");
5566                 }
5567         }
5568         return ret;
5569 }
5570
5571 /*
5572  * Mutex-protected thunk to flow_dv_translate().
5573  */
5574 static int
5575 flow_d_translate(struct rte_eth_dev *dev,
5576                  struct mlx5_flow *dev_flow,
5577                  const struct rte_flow_attr *attr,
5578                  const struct rte_flow_item items[],
5579                  const struct rte_flow_action actions[],
5580                  struct rte_flow_error *error)
5581 {
5582         int ret;
5583
5584         flow_d_shared_lock(dev);
5585         ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
5586         flow_d_shared_unlock(dev);
5587         return ret;
5588 }
5589
5590 /*
5591  * Mutex-protected thunk to flow_dv_apply().
5592  */
5593 static int
5594 flow_d_apply(struct rte_eth_dev *dev,
5595              struct rte_flow *flow,
5596              struct rte_flow_error *error)
5597 {
5598         int ret;
5599
5600         flow_d_shared_lock(dev);
5601         ret = flow_dv_apply(dev, flow, error);
5602         flow_d_shared_unlock(dev);
5603         return ret;
5604 }
5605
5606 /*
5607  * Mutex-protected thunk to flow_dv_remove().
5608  */
5609 static void
5610 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5611 {
5612         flow_d_shared_lock(dev);
5613         flow_dv_remove(dev, flow);
5614         flow_d_shared_unlock(dev);
5615 }
5616
5617 /*
5618  * Mutex-protected thunk to flow_dv_destroy().
5619  */
5620 static void
5621 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5622 {
5623         flow_d_shared_lock(dev);
5624         flow_dv_destroy(dev, flow);
5625         flow_d_shared_unlock(dev);
5626 }
5627
5628 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
5629         .validate = flow_dv_validate,
5630         .prepare = flow_dv_prepare,
5631         .translate = flow_d_translate,
5632         .apply = flow_d_apply,
5633         .remove = flow_d_remove,
5634         .destroy = flow_d_destroy,
5635         .query = flow_dv_query,
5636 };
5637
5638 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */