drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_prm.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 union flow_dv_attr {
  54         struct {
  55                 uint32_t valid:1;
  56                 uint32_t ipv4:1;
  57                 uint32_t ipv6:1;
  58                 uint32_t tcp:1;
  59                 uint32_t udp:1;
  60                 uint32_t reserved:27;
  61         };
  62         uint32_t attr;
  63 };
  64
  65 /**
  66  * Initialize flow attributes structure according to flow items' types.
  67  *
  68  * @param[in] item
  69  *   Pointer to item specification.
  70  * @param[out] attr
  71  *   Pointer to flow attributes structure.
  72  */
  73 static void
  74 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  75 {
  76         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  77                 switch (item->type) {
  78                 case RTE_FLOW_ITEM_TYPE_IPV4:
  79                         attr->ipv4 = 1;
  80                         break;
  81                 case RTE_FLOW_ITEM_TYPE_IPV6:
  82                         attr->ipv6 = 1;
  83                         break;
  84                 case RTE_FLOW_ITEM_TYPE_UDP:
  85                         attr->udp = 1;
  86                         break;
  87                 case RTE_FLOW_ITEM_TYPE_TCP:
  88                         attr->tcp = 1;
  89                         break;
  90                 default:
  91                         break;
  92                 }
  93         }
  94         attr->valid = 1;
  95 }
  96
  97 struct field_modify_info {
  98         uint32_t size; /* Size of field in protocol header, in bytes. */
  99         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 100         enum mlx5_modification_field id;
 101 };
 102
 103 struct field_modify_info modify_eth[] = {
 104         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 105         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 106         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 107         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 108         {0, 0, 0},
 109 };
 110
 111 struct field_modify_info modify_ipv4[] = {
 112         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 113         {4, 12, MLX5_MODI_OUT_SIPV4},
 114         {4, 16, MLX5_MODI_OUT_DIPV4},
 115         {0, 0, 0},
 116 };
 117
 118 struct field_modify_info modify_ipv6[] = {
 119         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 120         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 121         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 122         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 123         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 124         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 125         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 126         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 127         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 128         {0, 0, 0},
 129 };
 130
 131 struct field_modify_info modify_udp[] = {
 132         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 133         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 134         {0, 0, 0},
 135 };
 136
 137 struct field_modify_info modify_tcp[] = {
 138         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 139         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 140         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 141         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 142         {0, 0, 0},
 143 };
 144
 145 static void
 146 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 147                           uint8_t next_protocol, uint64_t *item_flags,
 148                           int *tunnel)
 149 {
 150         assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 151                item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 152         if (next_protocol == IPPROTO_IPIP) {
 153                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 154                 *tunnel = 1;
 155         }
 156         if (next_protocol == IPPROTO_IPV6) {
 157                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 158                 *tunnel = 1;
 159         }
 160 }
 161
 162 /**
 163  * Acquire the synchronizing object to protect multithreaded access
 164  * to shared dv context. Lock occurs only if context is actually
 165  * shared, i.e. we have multiport IB device and representors are
 166  * created.
 167  *
 168  * @param[in] dev
 169  *   Pointer to the rte_eth_dev structure.
 170  */
 171 static void
 172 flow_d_shared_lock(struct rte_eth_dev *dev)
 173 {
 174         struct mlx5_priv *priv = dev->data->dev_private;
 175         struct mlx5_ibv_shared *sh = priv->sh;
 176
 177         if (sh->dv_refcnt > 1) {
 178                 int ret;
 179
 180                 ret = pthread_mutex_lock(&sh->dv_mutex);
 181                 assert(!ret);
 182                 (void)ret;
 183         }
 184 }
 185
 186 static void
 187 flow_d_shared_unlock(struct rte_eth_dev *dev)
 188 {
 189         struct mlx5_priv *priv = dev->data->dev_private;
 190         struct mlx5_ibv_shared *sh = priv->sh;
 191
 192         if (sh->dv_refcnt > 1) {
 193                 int ret;
 194
 195                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 196                 assert(!ret);
 197                 (void)ret;
 198         }
 199 }
 200
 201 /**
 202  * Convert modify-header action to DV specification.
 203  *
 204  * @param[in] item
 205  *   Pointer to item specification.
 206  * @param[in] field
 207  *   Pointer to field modification information.
 208  * @param[in,out] resource
 209  *   Pointer to the modify-header resource.
 210  * @param[in] type
 211  *   Type of modification.
 212  * @param[out] error
 213  *   Pointer to the error structure.
 214  *
 215  * @return
 216  *   0 on success, a negative errno value otherwise and rte_errno is set.
 217  */
 218 static int
 219 flow_dv_convert_modify_action(struct rte_flow_item *item,
 220                               struct field_modify_info *field,
 221                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 222                               uint32_t type,
 223                               struct rte_flow_error *error)
 224 {
 225         uint32_t i = resource->actions_num;
 226         struct mlx5_modification_cmd *actions = resource->actions;
 227         const uint8_t *spec = item->spec;
 228         const uint8_t *mask = item->mask;
 229         uint32_t set;
 230
 231         while (field->size) {
 232                 set = 0;
 233                 /* Generate modify command for each mask segment. */
 234                 memcpy(&set, &mask[field->offset], field->size);
 235                 if (set) {
 236                         if (i >= MLX5_MODIFY_NUM)
 237                                 return rte_flow_error_set(error, EINVAL,
 238                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 239                                          "too many items to modify");
 240                         actions[i].action_type = type;
 241                         actions[i].field = field->id;
 242                         actions[i].length = field->size ==
 243                                         4 ? 0 : field->size * 8;
 244                         rte_memcpy(&actions[i].data[4 - field->size],
 245                                    &spec[field->offset], field->size);
 246                         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 247                         ++i;
 248                 }
 249                 if (resource->actions_num != i)
 250                         resource->actions_num = i;
 251                 field++;
 252         }
 253         if (!resource->actions_num)
 254                 return rte_flow_error_set(error, EINVAL,
 255                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 256                                           "invalid modification flow item");
 257         return 0;
 258 }
 259
 260 /**
 261  * Convert modify-header set IPv4 address action to DV specification.
 262  *
 263  * @param[in,out] resource
 264  *   Pointer to the modify-header resource.
 265  * @param[in] action
 266  *   Pointer to action specification.
 267  * @param[out] error
 268  *   Pointer to the error structure.
 269  *
 270  * @return
 271  *   0 on success, a negative errno value otherwise and rte_errno is set.
 272  */
 273 static int
 274 flow_dv_convert_action_modify_ipv4
 275                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 276                          const struct rte_flow_action *action,
 277                          struct rte_flow_error *error)
 278 {
 279         const struct rte_flow_action_set_ipv4 *conf =
 280                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 281         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 282         struct rte_flow_item_ipv4 ipv4;
 283         struct rte_flow_item_ipv4 ipv4_mask;
 284
 285         memset(&ipv4, 0, sizeof(ipv4));
 286         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 287         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 288                 ipv4.hdr.src_addr = conf->ipv4_addr;
 289                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 290         } else {
 291                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 292                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 293         }
 294         item.spec = &ipv4;
 295         item.mask = &ipv4_mask;
 296         return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
 297                                              MLX5_MODIFICATION_TYPE_SET, error);
 298 }
 299
 300 /**
 301  * Convert modify-header set IPv6 address action to DV specification.
 302  *
 303  * @param[in,out] resource
 304  *   Pointer to the modify-header resource.
 305  * @param[in] action
 306  *   Pointer to action specification.
 307  * @param[out] error
 308  *   Pointer to the error structure.
 309  *
 310  * @return
 311  *   0 on success, a negative errno value otherwise and rte_errno is set.
 312  */
 313 static int
 314 flow_dv_convert_action_modify_ipv6
 315                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 316                          const struct rte_flow_action *action,
 317                          struct rte_flow_error *error)
 318 {
 319         const struct rte_flow_action_set_ipv6 *conf =
 320                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 321         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 322         struct rte_flow_item_ipv6 ipv6;
 323         struct rte_flow_item_ipv6 ipv6_mask;
 324
 325         memset(&ipv6, 0, sizeof(ipv6));
 326         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 327         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 328                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 329                        sizeof(ipv6.hdr.src_addr));
 330                 memcpy(&ipv6_mask.hdr.src_addr,
 331                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 332                        sizeof(ipv6.hdr.src_addr));
 333         } else {
 334                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 335                        sizeof(ipv6.hdr.dst_addr));
 336                 memcpy(&ipv6_mask.hdr.dst_addr,
 337                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 338                        sizeof(ipv6.hdr.dst_addr));
 339         }
 340         item.spec = &ipv6;
 341         item.mask = &ipv6_mask;
 342         return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
 343                                              MLX5_MODIFICATION_TYPE_SET, error);
 344 }
 345
 346 /**
 347  * Convert modify-header set MAC address action to DV specification.
 348  *
 349  * @param[in,out] resource
 350  *   Pointer to the modify-header resource.
 351  * @param[in] action
 352  *   Pointer to action specification.
 353  * @param[out] error
 354  *   Pointer to the error structure.
 355  *
 356  * @return
 357  *   0 on success, a negative errno value otherwise and rte_errno is set.
 358  */
 359 static int
 360 flow_dv_convert_action_modify_mac
 361                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 362                          const struct rte_flow_action *action,
 363                          struct rte_flow_error *error)
 364 {
 365         const struct rte_flow_action_set_mac *conf =
 366                 (const struct rte_flow_action_set_mac *)(action->conf);
 367         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 368         struct rte_flow_item_eth eth;
 369         struct rte_flow_item_eth eth_mask;
 370
 371         memset(&eth, 0, sizeof(eth));
 372         memset(&eth_mask, 0, sizeof(eth_mask));
 373         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 374                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 375                        sizeof(eth.src.addr_bytes));
 376                 memcpy(&eth_mask.src.addr_bytes,
 377                        &rte_flow_item_eth_mask.src.addr_bytes,
 378                        sizeof(eth_mask.src.addr_bytes));
 379         } else {
 380                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 381                        sizeof(eth.dst.addr_bytes));
 382                 memcpy(&eth_mask.dst.addr_bytes,
 383                        &rte_flow_item_eth_mask.dst.addr_bytes,
 384                        sizeof(eth_mask.dst.addr_bytes));
 385         }
 386         item.spec = &eth;
 387         item.mask = &eth_mask;
 388         return flow_dv_convert_modify_action(&item, modify_eth, resource,
 389                                              MLX5_MODIFICATION_TYPE_SET, error);
 390 }
 391
 392 /**
 393  * Convert modify-header set TP action to DV specification.
 394  *
 395  * @param[in,out] resource
 396  *   Pointer to the modify-header resource.
 397  * @param[in] action
 398  *   Pointer to action specification.
 399  * @param[in] items
 400  *   Pointer to rte_flow_item objects list.
 401  * @param[in] attr
 402  *   Pointer to flow attributes structure.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   0 on success, a negative errno value otherwise and rte_errno is set.
 408  */
 409 static int
 410 flow_dv_convert_action_modify_tp
 411                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 412                          const struct rte_flow_action *action,
 413                          const struct rte_flow_item *items,
 414                          union flow_dv_attr *attr,
 415                          struct rte_flow_error *error)
 416 {
 417         const struct rte_flow_action_set_tp *conf =
 418                 (const struct rte_flow_action_set_tp *)(action->conf);
 419         struct rte_flow_item item;
 420         struct rte_flow_item_udp udp;
 421         struct rte_flow_item_udp udp_mask;
 422         struct rte_flow_item_tcp tcp;
 423         struct rte_flow_item_tcp tcp_mask;
 424         struct field_modify_info *field;
 425
 426         if (!attr->valid)
 427                 flow_dv_attr_init(items, attr);
 428         if (attr->udp) {
 429                 memset(&udp, 0, sizeof(udp));
 430                 memset(&udp_mask, 0, sizeof(udp_mask));
 431                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 432                         udp.hdr.src_port = conf->port;
 433                         udp_mask.hdr.src_port =
 434                                         rte_flow_item_udp_mask.hdr.src_port;
 435                 } else {
 436                         udp.hdr.dst_port = conf->port;
 437                         udp_mask.hdr.dst_port =
 438                                         rte_flow_item_udp_mask.hdr.dst_port;
 439                 }
 440                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 441                 item.spec = &udp;
 442                 item.mask = &udp_mask;
 443                 field = modify_udp;
 444         }
 445         if (attr->tcp) {
 446                 memset(&tcp, 0, sizeof(tcp));
 447                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 448                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 449                         tcp.hdr.src_port = conf->port;
 450                         tcp_mask.hdr.src_port =
 451                                         rte_flow_item_tcp_mask.hdr.src_port;
 452                 } else {
 453                         tcp.hdr.dst_port = conf->port;
 454                         tcp_mask.hdr.dst_port =
 455                                         rte_flow_item_tcp_mask.hdr.dst_port;
 456                 }
 457                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 458                 item.spec = &tcp;
 459                 item.mask = &tcp_mask;
 460                 field = modify_tcp;
 461         }
 462         return flow_dv_convert_modify_action(&item, field, resource,
 463                                              MLX5_MODIFICATION_TYPE_SET, error);
 464 }
 465
 466 /**
 467  * Convert modify-header set TTL action to DV specification.
 468  *
 469  * @param[in,out] resource
 470  *   Pointer to the modify-header resource.
 471  * @param[in] action
 472  *   Pointer to action specification.
 473  * @param[in] items
 474  *   Pointer to rte_flow_item objects list.
 475  * @param[in] attr
 476  *   Pointer to flow attributes structure.
 477  * @param[out] error
 478  *   Pointer to the error structure.
 479  *
 480  * @return
 481  *   0 on success, a negative errno value otherwise and rte_errno is set.
 482  */
 483 static int
 484 flow_dv_convert_action_modify_ttl
 485                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 486                          const struct rte_flow_action *action,
 487                          const struct rte_flow_item *items,
 488                          union flow_dv_attr *attr,
 489                          struct rte_flow_error *error)
 490 {
 491         const struct rte_flow_action_set_ttl *conf =
 492                 (const struct rte_flow_action_set_ttl *)(action->conf);
 493         struct rte_flow_item item;
 494         struct rte_flow_item_ipv4 ipv4;
 495         struct rte_flow_item_ipv4 ipv4_mask;
 496         struct rte_flow_item_ipv6 ipv6;
 497         struct rte_flow_item_ipv6 ipv6_mask;
 498         struct field_modify_info *field;
 499
 500         if (!attr->valid)
 501                 flow_dv_attr_init(items, attr);
 502         if (attr->ipv4) {
 503                 memset(&ipv4, 0, sizeof(ipv4));
 504                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 505                 ipv4.hdr.time_to_live = conf->ttl_value;
 506                 ipv4_mask.hdr.time_to_live = 0xFF;
 507                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 508                 item.spec = &ipv4;
 509                 item.mask = &ipv4_mask;
 510                 field = modify_ipv4;
 511         }
 512         if (attr->ipv6) {
 513                 memset(&ipv6, 0, sizeof(ipv6));
 514                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 515                 ipv6.hdr.hop_limits = conf->ttl_value;
 516                 ipv6_mask.hdr.hop_limits = 0xFF;
 517                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 518                 item.spec = &ipv6;
 519                 item.mask = &ipv6_mask;
 520                 field = modify_ipv6;
 521         }
 522         return flow_dv_convert_modify_action(&item, field, resource,
 523                                              MLX5_MODIFICATION_TYPE_SET, error);
 524 }
 525
 526 /**
 527  * Convert modify-header decrement TTL action to DV specification.
 528  *
 529  * @param[in,out] resource
 530  *   Pointer to the modify-header resource.
 531  * @param[in] action
 532  *   Pointer to action specification.
 533  * @param[in] items
 534  *   Pointer to rte_flow_item objects list.
 535  * @param[in] attr
 536  *   Pointer to flow attributes structure.
 537  * @param[out] error
 538  *   Pointer to the error structure.
 539  *
 540  * @return
 541  *   0 on success, a negative errno value otherwise and rte_errno is set.
 542  */
 543 static int
 544 flow_dv_convert_action_modify_dec_ttl
 545                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 546                          const struct rte_flow_item *items,
 547                          union flow_dv_attr *attr,
 548                          struct rte_flow_error *error)
 549 {
 550         struct rte_flow_item item;
 551         struct rte_flow_item_ipv4 ipv4;
 552         struct rte_flow_item_ipv4 ipv4_mask;
 553         struct rte_flow_item_ipv6 ipv6;
 554         struct rte_flow_item_ipv6 ipv6_mask;
 555         struct field_modify_info *field;
 556
 557         if (!attr->valid)
 558                 flow_dv_attr_init(items, attr);
 559         if (attr->ipv4) {
 560                 memset(&ipv4, 0, sizeof(ipv4));
 561                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 562                 ipv4.hdr.time_to_live = 0xFF;
 563                 ipv4_mask.hdr.time_to_live = 0xFF;
 564                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 565                 item.spec = &ipv4;
 566                 item.mask = &ipv4_mask;
 567                 field = modify_ipv4;
 568         }
 569         if (attr->ipv6) {
 570                 memset(&ipv6, 0, sizeof(ipv6));
 571                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 572                 ipv6.hdr.hop_limits = 0xFF;
 573                 ipv6_mask.hdr.hop_limits = 0xFF;
 574                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 575                 item.spec = &ipv6;
 576                 item.mask = &ipv6_mask;
 577                 field = modify_ipv6;
 578         }
 579         return flow_dv_convert_modify_action(&item, field, resource,
 580                                              MLX5_MODIFICATION_TYPE_ADD, error);
 581 }
 582
 583 /**
 584  * Convert modify-header increment/decrement TCP Sequence number
 585  * to DV specification.
 586  *
 587  * @param[in,out] resource
 588  *   Pointer to the modify-header resource.
 589  * @param[in] action
 590  *   Pointer to action specification.
 591  * @param[out] error
 592  *   Pointer to the error structure.
 593  *
 594  * @return
 595  *   0 on success, a negative errno value otherwise and rte_errno is set.
 596  */
 597 static int
 598 flow_dv_convert_action_modify_tcp_seq
 599                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 600                          const struct rte_flow_action *action,
 601                          struct rte_flow_error *error)
 602 {
 603         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 604         uint64_t value = rte_be_to_cpu_32(*conf);
 605         struct rte_flow_item item;
 606         struct rte_flow_item_tcp tcp;
 607         struct rte_flow_item_tcp tcp_mask;
 608
 609         memset(&tcp, 0, sizeof(tcp));
 610         memset(&tcp_mask, 0, sizeof(tcp_mask));
 611         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 612                 /*
 613                  * The HW has no decrement operation, only increment operation.
 614                  * To simulate decrement X from Y using increment operation
 615                  * we need to add UINT32_MAX X times to Y.
 616                  * Each adding of UINT32_MAX decrements Y by 1.
 617                  */
 618                 value *= UINT32_MAX;
 619         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 620         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 621         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 622         item.spec = &tcp;
 623         item.mask = &tcp_mask;
 624         return flow_dv_convert_modify_action(&item, modify_tcp, resource,
 625                                              MLX5_MODIFICATION_TYPE_ADD, error);
 626 }
 627
 628 /**
 629  * Convert modify-header increment/decrement TCP Acknowledgment number
 630  * to DV specification.
 631  *
 632  * @param[in,out] resource
 633  *   Pointer to the modify-header resource.
 634  * @param[in] action
 635  *   Pointer to action specification.
 636  * @param[out] error
 637  *   Pointer to the error structure.
 638  *
 639  * @return
 640  *   0 on success, a negative errno value otherwise and rte_errno is set.
 641  */
 642 static int
 643 flow_dv_convert_action_modify_tcp_ack
 644                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 645                          const struct rte_flow_action *action,
 646                          struct rte_flow_error *error)
 647 {
 648         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 649         uint64_t value = rte_be_to_cpu_32(*conf);
 650         struct rte_flow_item item;
 651         struct rte_flow_item_tcp tcp;
 652         struct rte_flow_item_tcp tcp_mask;
 653
 654         memset(&tcp, 0, sizeof(tcp));
 655         memset(&tcp_mask, 0, sizeof(tcp_mask));
 656         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 657                 /*
 658                  * The HW has no decrement operation, only increment operation.
 659                  * To simulate decrement X from Y using increment operation
 660                  * we need to add UINT32_MAX X times to Y.
 661                  * Each adding of UINT32_MAX decrements Y by 1.
 662                  */
 663                 value *= UINT32_MAX;
 664         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 665         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 666         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 667         item.spec = &tcp;
 668         item.mask = &tcp_mask;
 669         return flow_dv_convert_modify_action(&item, modify_tcp, resource,
 670                                              MLX5_MODIFICATION_TYPE_ADD, error);
 671 }
 672
 673 /**
 674  * Validate META item.
 675  *
 676  * @param[in] dev
 677  *   Pointer to the rte_eth_dev structure.
 678  * @param[in] item
 679  *   Item specification.
 680  * @param[in] attr
 681  *   Attributes of flow that includes this item.
 682  * @param[out] error
 683  *   Pointer to error structure.
 684  *
 685  * @return
 686  *   0 on success, a negative errno value otherwise and rte_errno is set.
 687  */
 688 static int
 689 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
 690                            const struct rte_flow_item *item,
 691                            const struct rte_flow_attr *attr,
 692                            struct rte_flow_error *error)
 693 {
 694         const struct rte_flow_item_meta *spec = item->spec;
 695         const struct rte_flow_item_meta *mask = item->mask;
 696         const struct rte_flow_item_meta nic_mask = {
 697                 .data = RTE_BE32(UINT32_MAX)
 698         };
 699         int ret;
 700         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
 701
 702         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
 703                 return rte_flow_error_set(error, EPERM,
 704                                           RTE_FLOW_ERROR_TYPE_ITEM,
 705                                           NULL,
 706                                           "match on metadata offload "
 707                                           "configuration is off for this port");
 708         if (!spec)
 709                 return rte_flow_error_set(error, EINVAL,
 710                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 711                                           item->spec,
 712                                           "data cannot be empty");
 713         if (!spec->data)
 714                 return rte_flow_error_set(error, EINVAL,
 715                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 716                                           NULL,
 717                                           "data cannot be zero");
 718         if (!mask)
 719                 mask = &rte_flow_item_meta_mask;
 720         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 721                                         (const uint8_t *)&nic_mask,
 722                                         sizeof(struct rte_flow_item_meta),
 723                                         error);
 724         if (ret < 0)
 725                 return ret;
 726         if (attr->ingress)
 727                 return rte_flow_error_set(error, ENOTSUP,
 728                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 729                                           NULL,
 730                                           "pattern not supported for ingress");
 731         return 0;
 732 }
 733
 734 /**
 735  * Validate vport item.
 736  *
 737  * @param[in] dev
 738  *   Pointer to the rte_eth_dev structure.
 739  * @param[in] item
 740  *   Item specification.
 741  * @param[in] attr
 742  *   Attributes of flow that includes this item.
 743  * @param[in] item_flags
 744  *   Bit-fields that holds the items detected until now.
 745  * @param[out] error
 746  *   Pointer to error structure.
 747  *
 748  * @return
 749  *   0 on success, a negative errno value otherwise and rte_errno is set.
 750  */
 751 static int
 752 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
 753                               const struct rte_flow_item *item,
 754                               const struct rte_flow_attr *attr,
 755                               uint64_t item_flags,
 756                               struct rte_flow_error *error)
 757 {
 758         const struct rte_flow_item_port_id *spec = item->spec;
 759         const struct rte_flow_item_port_id *mask = item->mask;
 760         const struct rte_flow_item_port_id switch_mask = {
 761                         .id = 0xffffffff,
 762         };
 763         uint16_t esw_domain_id;
 764         uint16_t item_port_esw_domain_id;
 765         int ret;
 766
 767         if (!attr->transfer)
 768                 return rte_flow_error_set(error, EINVAL,
 769                                           RTE_FLOW_ERROR_TYPE_ITEM,
 770                                           NULL,
 771                                           "match on port id is valid only"
 772                                           " when transfer flag is enabled");
 773         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
 774                 return rte_flow_error_set(error, ENOTSUP,
 775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
 776                                           "multiple source ports are not"
 777                                           " supported");
 778         if (!mask)
 779                 mask = &switch_mask;
 780         if (mask->id != 0xffffffff)
 781                 return rte_flow_error_set(error, ENOTSUP,
 782                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
 783                                            mask,
 784                                            "no support for partial mask on"
 785                                            " \"id\" field");
 786         ret = mlx5_flow_item_acceptable
 787                                 (item, (const uint8_t *)mask,
 788                                  (const uint8_t *)&rte_flow_item_port_id_mask,
 789                                  sizeof(struct rte_flow_item_port_id),
 790                                  error);
 791         if (ret)
 792                 return ret;
 793         if (!spec)
 794                 return 0;
 795         ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
 796                                         NULL);
 797         if (ret)
 798                 return rte_flow_error_set(error, -ret,
 799                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
 800                                           "failed to obtain E-Switch info for"
 801                                           " port");
 802         ret = mlx5_port_to_eswitch_info(dev->data->port_id,
 803                                         &esw_domain_id, NULL);
 804         if (ret < 0)
 805                 return rte_flow_error_set(error, -ret,
 806                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 807                                           NULL,
 808                                           "failed to obtain E-Switch info");
 809         if (item_port_esw_domain_id != esw_domain_id)
 810                 return rte_flow_error_set(error, -ret,
 811                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
 812                                           "cannot match on a port from a"
 813                                           " different E-Switch");
 814         return 0;
 815 }
 816
 817 /**
 818  * Validate the pop VLAN action.
 819  *
 820  * @param[in] action_flags
 821  *   Holds the actions detected until now.
 822  * @param[in] action
 823  *   Pointer to the pop vlan action.
 824  * @param[in] item_flags
 825  *   The items found in this flow rule.
 826  * @param[in] attr
 827  *   Pointer to flow attributes.
 828  * @param[out] error
 829  *   Pointer to error structure.
 830  *
 831  * @return
 832  *   0 on success, a negative errno value otherwise and rte_errno is set.
 833  */
 834 static int
 835 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
 836                                  uint64_t action_flags,
 837                                  const struct rte_flow_action *action,
 838                                  uint64_t item_flags,
 839                                  const struct rte_flow_attr *attr,
 840                                  struct rte_flow_error *error)
 841 {
 842         struct mlx5_priv *priv = dev->data->dev_private;
 843
 844         (void)action;
 845         (void)attr;
 846         if (!priv->sh->pop_vlan_action)
 847                 return rte_flow_error_set(error, ENOTSUP,
 848                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 849                                           NULL,
 850                                           "pop vlan action is not supported");
 851         /*
 852          * Check for inconsistencies:
 853          *  fail strip_vlan in a flow that matches packets without VLAN tags.
 854          *  fail strip_vlan in a flow that matches packets without explicitly a
 855          *  matching on VLAN tag ?
 856          */
 857         if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN)
 858                 return rte_flow_error_set(error, ENOTSUP,
 859                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 860                                           NULL,
 861                                           "no support for multiple vlan pop "
 862                                           "actions");
 863         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
 864                 return rte_flow_error_set(error, ENOTSUP,
 865                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 866                                           NULL,
 867                                           "cannot pop vlan without a "
 868                                           "match on (outer) vlan in the flow");
 869         return 0;
 870 }
 871
 872 /**
 873  * Validate count action.
 874  *
 875  * @param[in] dev
 876  *   device otr.
 877  * @param[out] error
 878  *   Pointer to error structure.
 879  *
 880  * @return
 881  *   0 on success, a negative errno value otherwise and rte_errno is set.
 882  */
 883 static int
 884 flow_dv_validate_action_count(struct rte_eth_dev *dev,
 885                               struct rte_flow_error *error)
 886 {
 887         struct mlx5_priv *priv = dev->data->dev_private;
 888
 889         if (!priv->config.devx)
 890                 goto notsup_err;
 891 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
 892         return 0;
 893 #endif
 894 notsup_err:
 895         return rte_flow_error_set
 896                       (error, ENOTSUP,
 897                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 898                        NULL,
 899                        "count action not supported");
 900 }
 901
 902 /**
 903  * Validate the L2 encap action.
 904  *
 905  * @param[in] action_flags
 906  *   Holds the actions detected until now.
 907  * @param[in] action
 908  *   Pointer to the encap action.
 909  * @param[in] attr
 910  *   Pointer to flow attributes
 911  * @param[out] error
 912  *   Pointer to error structure.
 913  *
 914  * @return
 915  *   0 on success, a negative errno value otherwise and rte_errno is set.
 916  */
 917 static int
 918 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 919                                  const struct rte_flow_action *action,
 920                                  const struct rte_flow_attr *attr,
 921                                  struct rte_flow_error *error)
 922 {
 923         if (!(action->conf))
 924                 return rte_flow_error_set(error, EINVAL,
 925                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 926                                           "configuration cannot be null");
 927         if (action_flags & MLX5_FLOW_ACTION_DROP)
 928                 return rte_flow_error_set(error, EINVAL,
 929                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 930                                           "can't drop and encap in same flow");
 931         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 932                 return rte_flow_error_set(error, EINVAL,
 933                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 934                                           "can only have a single encap or"
 935                                           " decap action in a flow");
 936         if (!attr->transfer && attr->ingress)
 937                 return rte_flow_error_set(error, ENOTSUP,
 938                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 939                                           NULL,
 940                                           "encap action not supported for "
 941                                           "ingress");
 942         return 0;
 943 }
 944
 945 /**
 946  * Validate the L2 decap action.
 947  *
 948  * @param[in] action_flags
 949  *   Holds the actions detected until now.
 950  * @param[in] attr
 951  *   Pointer to flow attributes
 952  * @param[out] error
 953  *   Pointer to error structure.
 954  *
 955  * @return
 956  *   0 on success, a negative errno value otherwise and rte_errno is set.
 957  */
 958 static int
 959 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 960                                  const struct rte_flow_attr *attr,
 961                                  struct rte_flow_error *error)
 962 {
 963         if (action_flags & MLX5_FLOW_ACTION_DROP)
 964                 return rte_flow_error_set(error, EINVAL,
 965                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 966                                           "can't drop and decap in same flow");
 967         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 968                 return rte_flow_error_set(error, EINVAL,
 969                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 970                                           "can only have a single encap or"
 971                                           " decap action in a flow");
 972         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 973                 return rte_flow_error_set(error, EINVAL,
 974                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 975                                           "can't have decap action after"
 976                                           " modify action");
 977         if (attr->egress)
 978                 return rte_flow_error_set(error, ENOTSUP,
 979                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 980                                           NULL,
 981                                           "decap action not supported for "
 982                                           "egress");
 983         return 0;
 984 }
 985
 986 /**
 987  * Validate the raw encap action.
 988  *
 989  * @param[in] action_flags
 990  *   Holds the actions detected until now.
 991  * @param[in] action
 992  *   Pointer to the encap action.
 993  * @param[in] attr
 994  *   Pointer to flow attributes
 995  * @param[out] error
 996  *   Pointer to error structure.
 997  *
 998  * @return
 999  *   0 on success, a negative errno value otherwise and rte_errno is set.
1000  */
1001 static int
1002 flow_dv_validate_action_raw_encap(uint64_t action_flags,
1003                                   const struct rte_flow_action *action,
1004                                   const struct rte_flow_attr *attr,
1005                                   struct rte_flow_error *error)
1006 {
1007         const struct rte_flow_action_raw_encap *raw_encap =
1008                 (const struct rte_flow_action_raw_encap *)action->conf;
1009         if (!(action->conf))
1010                 return rte_flow_error_set(error, EINVAL,
1011                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1012                                           "configuration cannot be null");
1013         if (action_flags & MLX5_FLOW_ACTION_DROP)
1014                 return rte_flow_error_set(error, EINVAL,
1015                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1016                                           "can't drop and encap in same flow");
1017         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1018                 return rte_flow_error_set(error, EINVAL,
1019                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1020                                           "can only have a single encap"
1021                                           " action in a flow");
1022         /* encap without preceding decap is not supported for ingress */
1023         if (!attr->transfer &&  attr->ingress &&
1024             !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
1025                 return rte_flow_error_set(error, ENOTSUP,
1026                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1027                                           NULL,
1028                                           "encap action not supported for "
1029                                           "ingress");
1030         if (!raw_encap->size || !raw_encap->data)
1031                 return rte_flow_error_set(error, EINVAL,
1032                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1033                                           "raw encap data cannot be empty");
1034         return 0;
1035 }
1036
1037 /**
1038  * Validate the raw decap action.
1039  *
1040  * @param[in] action_flags
1041  *   Holds the actions detected until now.
1042  * @param[in] action
1043  *   Pointer to the encap action.
1044  * @param[in] attr
1045  *   Pointer to flow attributes
1046  * @param[out] error
1047  *   Pointer to error structure.
1048  *
1049  * @return
1050  *   0 on success, a negative errno value otherwise and rte_errno is set.
1051  */
1052 static int
1053 flow_dv_validate_action_raw_decap(uint64_t action_flags,
1054                                   const struct rte_flow_action *action,
1055                                   const struct rte_flow_attr *attr,
1056                                   struct rte_flow_error *error)
1057 {
1058         if (action_flags & MLX5_FLOW_ACTION_DROP)
1059                 return rte_flow_error_set(error, EINVAL,
1060                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1061                                           "can't drop and decap in same flow");
1062         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1063                 return rte_flow_error_set(error, EINVAL,
1064                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1065                                           "can't have encap action before"
1066                                           " decap action");
1067         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1068                 return rte_flow_error_set(error, EINVAL,
1069                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1070                                           "can only have a single decap"
1071                                           " action in a flow");
1072         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1073                 return rte_flow_error_set(error, EINVAL,
1074                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1075                                           "can't have decap action after"
1076                                           " modify action");
1077         /* decap action is valid on egress only if it is followed by encap */
1078         if (attr->egress) {
1079                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1080                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1081                        action++) {
1082                 }
1083                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1084                         return rte_flow_error_set
1085                                         (error, ENOTSUP,
1086                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1087                                          NULL, "decap action not supported"
1088                                          " for egress");
1089         }
1090         return 0;
1091 }
1092
1093 /**
1094  * Find existing encap/decap resource or create and register a new one.
1095  *
1096  * @param dev[in, out]
1097  *   Pointer to rte_eth_dev structure.
1098  * @param[in, out] resource
1099  *   Pointer to encap/decap resource.
1100  * @parm[in, out] dev_flow
1101  *   Pointer to the dev_flow.
1102  * @param[out] error
1103  *   pointer to error structure.
1104  *
1105  * @return
1106  *   0 on success otherwise -errno and errno is set.
1107  */
1108 static int
1109 flow_dv_encap_decap_resource_register
1110                         (struct rte_eth_dev *dev,
1111                          struct mlx5_flow_dv_encap_decap_resource *resource,
1112                          struct mlx5_flow *dev_flow,
1113                          struct rte_flow_error *error)
1114 {
1115         struct mlx5_priv *priv = dev->data->dev_private;
1116         struct mlx5_ibv_shared *sh = priv->sh;
1117         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1118         struct rte_flow *flow = dev_flow->flow;
1119         struct mlx5dv_dr_domain *domain;
1120
1121         resource->flags = flow->group ? 0 : 1;
1122         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1123                 domain = sh->fdb_domain;
1124         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1125                 domain = sh->rx_domain;
1126         else
1127                 domain = sh->tx_domain;
1128
1129         /* Lookup a matching resource from cache. */
1130         LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1131                 if (resource->reformat_type == cache_resource->reformat_type &&
1132                     resource->ft_type == cache_resource->ft_type &&
1133                     resource->flags == cache_resource->flags &&
1134                     resource->size == cache_resource->size &&
1135                     !memcmp((const void *)resource->buf,
1136                             (const void *)cache_resource->buf,
1137                             resource->size)) {
1138                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1139                                 (void *)cache_resource,
1140                                 rte_atomic32_read(&cache_resource->refcnt));
1141                         rte_atomic32_inc(&cache_resource->refcnt);
1142                         dev_flow->dv.encap_decap = cache_resource;
1143                         return 0;
1144                 }
1145         }
1146         /* Register new encap/decap resource. */
1147         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1148         if (!cache_resource)
1149                 return rte_flow_error_set(error, ENOMEM,
1150                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1151                                           "cannot allocate resource memory");
1152         *cache_resource = *resource;
1153         cache_resource->verbs_action =
1154                 mlx5_glue->dv_create_flow_action_packet_reformat
1155                         (sh->ctx, cache_resource->reformat_type,
1156                          cache_resource->ft_type, domain, cache_resource->flags,
1157                          cache_resource->size,
1158                          (cache_resource->size ? cache_resource->buf : NULL));
1159         if (!cache_resource->verbs_action) {
1160                 rte_free(cache_resource);
1161                 return rte_flow_error_set(error, ENOMEM,
1162                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1163                                           NULL, "cannot create action");
1164         }
1165         rte_atomic32_init(&cache_resource->refcnt);
1166         rte_atomic32_inc(&cache_resource->refcnt);
1167         LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1168         dev_flow->dv.encap_decap = cache_resource;
1169         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1170                 (void *)cache_resource,
1171                 rte_atomic32_read(&cache_resource->refcnt));
1172         return 0;
1173 }
1174
1175 /**
1176  * Find existing table jump resource or create and register a new one.
1177  *
1178  * @param dev[in, out]
1179  *   Pointer to rte_eth_dev structure.
1180  * @param[in, out] resource
1181  *   Pointer to jump table resource.
1182  * @parm[in, out] dev_flow
1183  *   Pointer to the dev_flow.
1184  * @param[out] error
1185  *   pointer to error structure.
1186  *
1187  * @return
1188  *   0 on success otherwise -errno and errno is set.
1189  */
1190 static int
1191 flow_dv_jump_tbl_resource_register
1192                         (struct rte_eth_dev *dev,
1193                          struct mlx5_flow_dv_jump_tbl_resource *resource,
1194                          struct mlx5_flow *dev_flow,
1195                          struct rte_flow_error *error)
1196 {
1197         struct mlx5_priv *priv = dev->data->dev_private;
1198         struct mlx5_ibv_shared *sh = priv->sh;
1199         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1200
1201         /* Lookup a matching resource from cache. */
1202         LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1203                 if (resource->tbl == cache_resource->tbl) {
1204                         DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1205                                 (void *)cache_resource,
1206                                 rte_atomic32_read(&cache_resource->refcnt));
1207                         rte_atomic32_inc(&cache_resource->refcnt);
1208                         dev_flow->dv.jump = cache_resource;
1209                         return 0;
1210                 }
1211         }
1212         /* Register new jump table resource. */
1213         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1214         if (!cache_resource)
1215                 return rte_flow_error_set(error, ENOMEM,
1216                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1217                                           "cannot allocate resource memory");
1218         *cache_resource = *resource;
1219         cache_resource->action =
1220                 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1221                 (resource->tbl->obj);
1222         if (!cache_resource->action) {
1223                 rte_free(cache_resource);
1224                 return rte_flow_error_set(error, ENOMEM,
1225                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1226                                           NULL, "cannot create action");
1227         }
1228         rte_atomic32_init(&cache_resource->refcnt);
1229         rte_atomic32_inc(&cache_resource->refcnt);
1230         LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1231         dev_flow->dv.jump = cache_resource;
1232         DRV_LOG(DEBUG, "new jump table  resource %p: refcnt %d++",
1233                 (void *)cache_resource,
1234                 rte_atomic32_read(&cache_resource->refcnt));
1235         return 0;
1236 }
1237
1238 /**
1239  * Find existing table port ID resource or create and register a new one.
1240  *
1241  * @param dev[in, out]
1242  *   Pointer to rte_eth_dev structure.
1243  * @param[in, out] resource
1244  *   Pointer to port ID action resource.
1245  * @parm[in, out] dev_flow
1246  *   Pointer to the dev_flow.
1247  * @param[out] error
1248  *   pointer to error structure.
1249  *
1250  * @return
1251  *   0 on success otherwise -errno and errno is set.
1252  */
1253 static int
1254 flow_dv_port_id_action_resource_register
1255                         (struct rte_eth_dev *dev,
1256                          struct mlx5_flow_dv_port_id_action_resource *resource,
1257                          struct mlx5_flow *dev_flow,
1258                          struct rte_flow_error *error)
1259 {
1260         struct mlx5_priv *priv = dev->data->dev_private;
1261         struct mlx5_ibv_shared *sh = priv->sh;
1262         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1263
1264         /* Lookup a matching resource from cache. */
1265         LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1266                 if (resource->port_id == cache_resource->port_id) {
1267                         DRV_LOG(DEBUG, "port id action resource resource %p: "
1268                                 "refcnt %d++",
1269                                 (void *)cache_resource,
1270                                 rte_atomic32_read(&cache_resource->refcnt));
1271                         rte_atomic32_inc(&cache_resource->refcnt);
1272                         dev_flow->dv.port_id_action = cache_resource;
1273                         return 0;
1274                 }
1275         }
1276         /* Register new port id action resource. */
1277         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1278         if (!cache_resource)
1279                 return rte_flow_error_set(error, ENOMEM,
1280                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1281                                           "cannot allocate resource memory");
1282         *cache_resource = *resource;
1283         cache_resource->action =
1284                 mlx5_glue->dr_create_flow_action_dest_vport
1285                         (priv->sh->fdb_domain, resource->port_id);
1286         if (!cache_resource->action) {
1287                 rte_free(cache_resource);
1288                 return rte_flow_error_set(error, ENOMEM,
1289                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1290                                           NULL, "cannot create action");
1291         }
1292         rte_atomic32_init(&cache_resource->refcnt);
1293         rte_atomic32_inc(&cache_resource->refcnt);
1294         LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1295         dev_flow->dv.port_id_action = cache_resource;
1296         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1297                 (void *)cache_resource,
1298                 rte_atomic32_read(&cache_resource->refcnt));
1299         return 0;
1300 }
1301
1302 /**
1303  * Get the size of specific rte_flow_item_type
1304  *
1305  * @param[in] item_type
1306  *   Tested rte_flow_item_type.
1307  *
1308  * @return
1309  *   sizeof struct item_type, 0 if void or irrelevant.
1310  */
1311 static size_t
1312 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1313 {
1314         size_t retval;
1315
1316         switch (item_type) {
1317         case RTE_FLOW_ITEM_TYPE_ETH:
1318                 retval = sizeof(struct rte_flow_item_eth);
1319                 break;
1320         case RTE_FLOW_ITEM_TYPE_VLAN:
1321                 retval = sizeof(struct rte_flow_item_vlan);
1322                 break;
1323         case RTE_FLOW_ITEM_TYPE_IPV4:
1324                 retval = sizeof(struct rte_flow_item_ipv4);
1325                 break;
1326         case RTE_FLOW_ITEM_TYPE_IPV6:
1327                 retval = sizeof(struct rte_flow_item_ipv6);
1328                 break;
1329         case RTE_FLOW_ITEM_TYPE_UDP:
1330                 retval = sizeof(struct rte_flow_item_udp);
1331                 break;
1332         case RTE_FLOW_ITEM_TYPE_TCP:
1333                 retval = sizeof(struct rte_flow_item_tcp);
1334                 break;
1335         case RTE_FLOW_ITEM_TYPE_VXLAN:
1336                 retval = sizeof(struct rte_flow_item_vxlan);
1337                 break;
1338         case RTE_FLOW_ITEM_TYPE_GRE:
1339                 retval = sizeof(struct rte_flow_item_gre);
1340                 break;
1341         case RTE_FLOW_ITEM_TYPE_NVGRE:
1342                 retval = sizeof(struct rte_flow_item_nvgre);
1343                 break;
1344         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1345                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1346                 break;
1347         case RTE_FLOW_ITEM_TYPE_MPLS:
1348                 retval = sizeof(struct rte_flow_item_mpls);
1349                 break;
1350         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1351         default:
1352                 retval = 0;
1353                 break;
1354         }
1355         return retval;
1356 }
1357
1358 #define MLX5_ENCAP_IPV4_VERSION         0x40
1359 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
1360 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
1361 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
1362 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
1363 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
1364 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
1365
1366 /**
1367  * Convert the encap action data from list of rte_flow_item to raw buffer
1368  *
1369  * @param[in] items
1370  *   Pointer to rte_flow_item objects list.
1371  * @param[out] buf
1372  *   Pointer to the output buffer.
1373  * @param[out] size
1374  *   Pointer to the output buffer size.
1375  * @param[out] error
1376  *   Pointer to the error structure.
1377  *
1378  * @return
1379  *   0 on success, a negative errno value otherwise and rte_errno is set.
1380  */
1381 static int
1382 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1383                            size_t *size, struct rte_flow_error *error)
1384 {
1385         struct rte_ether_hdr *eth = NULL;
1386         struct rte_vlan_hdr *vlan = NULL;
1387         struct rte_ipv4_hdr *ipv4 = NULL;
1388         struct rte_ipv6_hdr *ipv6 = NULL;
1389         struct rte_udp_hdr *udp = NULL;
1390         struct rte_vxlan_hdr *vxlan = NULL;
1391         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1392         struct rte_gre_hdr *gre = NULL;
1393         size_t len;
1394         size_t temp_size = 0;
1395
1396         if (!items)
1397                 return rte_flow_error_set(error, EINVAL,
1398                                           RTE_FLOW_ERROR_TYPE_ACTION,
1399                                           NULL, "invalid empty data");
1400         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1401                 len = flow_dv_get_item_len(items->type);
1402                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1403                         return rte_flow_error_set(error, EINVAL,
1404                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1405                                                   (void *)items->type,
1406                                                   "items total size is too big"
1407                                                   " for encap action");
1408                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1409                 switch (items->type) {
1410                 case RTE_FLOW_ITEM_TYPE_ETH:
1411                         eth = (struct rte_ether_hdr *)&buf[temp_size];
1412                         break;
1413                 case RTE_FLOW_ITEM_TYPE_VLAN:
1414                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1415                         if (!eth)
1416                                 return rte_flow_error_set(error, EINVAL,
1417                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1418                                                 (void *)items->type,
1419                                                 "eth header not found");
1420                         if (!eth->ether_type)
1421                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1422                         break;
1423                 case RTE_FLOW_ITEM_TYPE_IPV4:
1424                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1425                         if (!vlan && !eth)
1426                                 return rte_flow_error_set(error, EINVAL,
1427                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1428                                                 (void *)items->type,
1429                                                 "neither eth nor vlan"
1430                                                 " header found");
1431                         if (vlan && !vlan->eth_proto)
1432                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1433                         else if (eth && !eth->ether_type)
1434                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1435                         if (!ipv4->version_ihl)
1436                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1437                                                     MLX5_ENCAP_IPV4_IHL_MIN;
1438                         if (!ipv4->time_to_live)
1439                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1440                         break;
1441                 case RTE_FLOW_ITEM_TYPE_IPV6:
1442                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1443                         if (!vlan && !eth)
1444                                 return rte_flow_error_set(error, EINVAL,
1445                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1446                                                 (void *)items->type,
1447                                                 "neither eth nor vlan"
1448                                                 " header found");
1449                         if (vlan && !vlan->eth_proto)
1450                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1451                         else if (eth && !eth->ether_type)
1452                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1453                         if (!ipv6->vtc_flow)
1454                                 ipv6->vtc_flow =
1455                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1456                         if (!ipv6->hop_limits)
1457                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1458                         break;
1459                 case RTE_FLOW_ITEM_TYPE_UDP:
1460                         udp = (struct rte_udp_hdr *)&buf[temp_size];
1461                         if (!ipv4 && !ipv6)
1462                                 return rte_flow_error_set(error, EINVAL,
1463                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1464                                                 (void *)items->type,
1465                                                 "ip header not found");
1466                         if (ipv4 && !ipv4->next_proto_id)
1467                                 ipv4->next_proto_id = IPPROTO_UDP;
1468                         else if (ipv6 && !ipv6->proto)
1469                                 ipv6->proto = IPPROTO_UDP;
1470                         break;
1471                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1472                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1473                         if (!udp)
1474                                 return rte_flow_error_set(error, EINVAL,
1475                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1476                                                 (void *)items->type,
1477                                                 "udp header not found");
1478                         if (!udp->dst_port)
1479                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1480                         if (!vxlan->vx_flags)
1481                                 vxlan->vx_flags =
1482                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1483                         break;
1484                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1485                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1486                         if (!udp)
1487                                 return rte_flow_error_set(error, EINVAL,
1488                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1489                                                 (void *)items->type,
1490                                                 "udp header not found");
1491                         if (!vxlan_gpe->proto)
1492                                 return rte_flow_error_set(error, EINVAL,
1493                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1494                                                 (void *)items->type,
1495                                                 "next protocol not found");
1496                         if (!udp->dst_port)
1497                                 udp->dst_port =
1498                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1499                         if (!vxlan_gpe->vx_flags)
1500                                 vxlan_gpe->vx_flags =
1501                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1502                         break;
1503                 case RTE_FLOW_ITEM_TYPE_GRE:
1504                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1505                         gre = (struct rte_gre_hdr *)&buf[temp_size];
1506                         if (!gre->proto)
1507                                 return rte_flow_error_set(error, EINVAL,
1508                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1509                                                 (void *)items->type,
1510                                                 "next protocol not found");
1511                         if (!ipv4 && !ipv6)
1512                                 return rte_flow_error_set(error, EINVAL,
1513                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1514                                                 (void *)items->type,
1515                                                 "ip header not found");
1516                         if (ipv4 && !ipv4->next_proto_id)
1517                                 ipv4->next_proto_id = IPPROTO_GRE;
1518                         else if (ipv6 && !ipv6->proto)
1519                                 ipv6->proto = IPPROTO_GRE;
1520                         break;
1521                 case RTE_FLOW_ITEM_TYPE_VOID:
1522                         break;
1523                 default:
1524                         return rte_flow_error_set(error, EINVAL,
1525                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1526                                                   (void *)items->type,
1527                                                   "unsupported item type");
1528                         break;
1529                 }
1530                 temp_size += len;
1531         }
1532         *size = temp_size;
1533         return 0;
1534 }
1535
1536 static int
1537 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1538 {
1539         struct rte_ether_hdr *eth = NULL;
1540         struct rte_vlan_hdr *vlan = NULL;
1541         struct rte_ipv6_hdr *ipv6 = NULL;
1542         struct rte_udp_hdr *udp = NULL;
1543         char *next_hdr;
1544         uint16_t proto;
1545
1546         eth = (struct rte_ether_hdr *)data;
1547         next_hdr = (char *)(eth + 1);
1548         proto = RTE_BE16(eth->ether_type);
1549
1550         /* VLAN skipping */
1551         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1552                 vlan = (struct rte_vlan_hdr *)next_hdr;
1553                 proto = RTE_BE16(vlan->eth_proto);
1554                 next_hdr += sizeof(struct rte_vlan_hdr);
1555         }
1556
1557         /* HW calculates IPv4 csum. no need to proceed */
1558         if (proto == RTE_ETHER_TYPE_IPV4)
1559                 return 0;
1560
1561         /* non IPv4/IPv6 header. not supported */
1562         if (proto != RTE_ETHER_TYPE_IPV6) {
1563                 return rte_flow_error_set(error, ENOTSUP,
1564                                           RTE_FLOW_ERROR_TYPE_ACTION,
1565                                           NULL, "Cannot offload non IPv4/IPv6");
1566         }
1567
1568         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1569
1570         /* ignore non UDP */
1571         if (ipv6->proto != IPPROTO_UDP)
1572                 return 0;
1573
1574         udp = (struct rte_udp_hdr *)(ipv6 + 1);
1575         udp->dgram_cksum = 0;
1576
1577         return 0;
1578 }
1579
1580 /**
1581  * Convert L2 encap action to DV specification.
1582  *
1583  * @param[in] dev
1584  *   Pointer to rte_eth_dev structure.
1585  * @param[in] action
1586  *   Pointer to action structure.
1587  * @param[in, out] dev_flow
1588  *   Pointer to the mlx5_flow.
1589  * @param[in] transfer
1590  *   Mark if the flow is E-Switch flow.
1591  * @param[out] error
1592  *   Pointer to the error structure.
1593  *
1594  * @return
1595  *   0 on success, a negative errno value otherwise and rte_errno is set.
1596  */
1597 static int
1598 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1599                                const struct rte_flow_action *action,
1600                                struct mlx5_flow *dev_flow,
1601                                uint8_t transfer,
1602                                struct rte_flow_error *error)
1603 {
1604         const struct rte_flow_item *encap_data;
1605         const struct rte_flow_action_raw_encap *raw_encap_data;
1606         struct mlx5_flow_dv_encap_decap_resource res = {
1607                 .reformat_type =
1608                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1609                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1610                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1611         };
1612
1613         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1614                 raw_encap_data =
1615                         (const struct rte_flow_action_raw_encap *)action->conf;
1616                 res.size = raw_encap_data->size;
1617                 memcpy(res.buf, raw_encap_data->data, res.size);
1618                 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1619                         return -rte_errno;
1620         } else {
1621                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1622                         encap_data =
1623                                 ((const struct rte_flow_action_vxlan_encap *)
1624                                                 action->conf)->definition;
1625                 else
1626                         encap_data =
1627                                 ((const struct rte_flow_action_nvgre_encap *)
1628                                                 action->conf)->definition;
1629                 if (flow_dv_convert_encap_data(encap_data, res.buf,
1630                                                &res.size, error))
1631                         return -rte_errno;
1632         }
1633         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1634                 return rte_flow_error_set(error, EINVAL,
1635                                           RTE_FLOW_ERROR_TYPE_ACTION,
1636                                           NULL, "can't create L2 encap action");
1637         return 0;
1638 }
1639
1640 /**
1641  * Convert L2 decap action to DV specification.
1642  *
1643  * @param[in] dev
1644  *   Pointer to rte_eth_dev structure.
1645  * @param[in, out] dev_flow
1646  *   Pointer to the mlx5_flow.
1647  * @param[in] transfer
1648  *   Mark if the flow is E-Switch flow.
1649  * @param[out] error
1650  *   Pointer to the error structure.
1651  *
1652  * @return
1653  *   0 on success, a negative errno value otherwise and rte_errno is set.
1654  */
1655 static int
1656 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1657                                struct mlx5_flow *dev_flow,
1658                                uint8_t transfer,
1659                                struct rte_flow_error *error)
1660 {
1661         struct mlx5_flow_dv_encap_decap_resource res = {
1662                 .size = 0,
1663                 .reformat_type =
1664                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1665                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1666                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1667         };
1668
1669         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1670                 return rte_flow_error_set(error, EINVAL,
1671                                           RTE_FLOW_ERROR_TYPE_ACTION,
1672                                           NULL, "can't create L2 decap action");
1673         return 0;
1674 }
1675
1676 /**
1677  * Convert raw decap/encap (L3 tunnel) action to DV specification.
1678  *
1679  * @param[in] dev
1680  *   Pointer to rte_eth_dev structure.
1681  * @param[in] action
1682  *   Pointer to action structure.
1683  * @param[in, out] dev_flow
1684  *   Pointer to the mlx5_flow.
1685  * @param[in] attr
1686  *   Pointer to the flow attributes.
1687  * @param[out] error
1688  *   Pointer to the error structure.
1689  *
1690  * @return
1691  *   0 on success, a negative errno value otherwise and rte_errno is set.
1692  */
1693 static int
1694 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1695                                 const struct rte_flow_action *action,
1696                                 struct mlx5_flow *dev_flow,
1697                                 const struct rte_flow_attr *attr,
1698                                 struct rte_flow_error *error)
1699 {
1700         const struct rte_flow_action_raw_encap *encap_data;
1701         struct mlx5_flow_dv_encap_decap_resource res;
1702
1703         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1704         res.size = encap_data->size;
1705         memcpy(res.buf, encap_data->data, res.size);
1706         res.reformat_type = attr->egress ?
1707                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1708                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1709         if (attr->transfer)
1710                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1711         else
1712                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1713                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1714         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1715                 return rte_flow_error_set(error, EINVAL,
1716                                           RTE_FLOW_ERROR_TYPE_ACTION,
1717                                           NULL, "can't create encap action");
1718         return 0;
1719 }
1720
1721 /**
1722  * Validate the modify-header actions.
1723  *
1724  * @param[in] action_flags
1725  *   Holds the actions detected until now.
1726  * @param[in] action
1727  *   Pointer to the modify action.
1728  * @param[out] error
1729  *   Pointer to error structure.
1730  *
1731  * @return
1732  *   0 on success, a negative errno value otherwise and rte_errno is set.
1733  */
1734 static int
1735 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1736                                    const struct rte_flow_action *action,
1737                                    struct rte_flow_error *error)
1738 {
1739         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1740                 return rte_flow_error_set(error, EINVAL,
1741                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1742                                           NULL, "action configuration not set");
1743         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1744                 return rte_flow_error_set(error, EINVAL,
1745                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1746                                           "can't have encap action before"
1747                                           " modify action");
1748         return 0;
1749 }
1750
1751 /**
1752  * Validate the modify-header MAC address actions.
1753  *
1754  * @param[in] action_flags
1755  *   Holds the actions detected until now.
1756  * @param[in] action
1757  *   Pointer to the modify action.
1758  * @param[in] item_flags
1759  *   Holds the items detected.
1760  * @param[out] error
1761  *   Pointer to error structure.
1762  *
1763  * @return
1764  *   0 on success, a negative errno value otherwise and rte_errno is set.
1765  */
1766 static int
1767 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1768                                    const struct rte_flow_action *action,
1769                                    const uint64_t item_flags,
1770                                    struct rte_flow_error *error)
1771 {
1772         int ret = 0;
1773
1774         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1775         if (!ret) {
1776                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1777                         return rte_flow_error_set(error, EINVAL,
1778                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1779                                                   NULL,
1780                                                   "no L2 item in pattern");
1781         }
1782         return ret;
1783 }
1784
1785 /**
1786  * Validate the modify-header IPv4 address actions.
1787  *
1788  * @param[in] action_flags
1789  *   Holds the actions detected until now.
1790  * @param[in] action
1791  *   Pointer to the modify action.
1792  * @param[in] item_flags
1793  *   Holds the items detected.
1794  * @param[out] error
1795  *   Pointer to error structure.
1796  *
1797  * @return
1798  *   0 on success, a negative errno value otherwise and rte_errno is set.
1799  */
1800 static int
1801 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
1802                                     const struct rte_flow_action *action,
1803                                     const uint64_t item_flags,
1804                                     struct rte_flow_error *error)
1805 {
1806         int ret = 0;
1807
1808         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1809         if (!ret) {
1810                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
1811                         return rte_flow_error_set(error, EINVAL,
1812                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1813                                                   NULL,
1814                                                   "no ipv4 item in pattern");
1815         }
1816         return ret;
1817 }
1818
1819 /**
1820  * Validate the modify-header IPv6 address actions.
1821  *
1822  * @param[in] action_flags
1823  *   Holds the actions detected until now.
1824  * @param[in] action
1825  *   Pointer to the modify action.
1826  * @param[in] item_flags
1827  *   Holds the items detected.
1828  * @param[out] error
1829  *   Pointer to error structure.
1830  *
1831  * @return
1832  *   0 on success, a negative errno value otherwise and rte_errno is set.
1833  */
1834 static int
1835 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
1836                                     const struct rte_flow_action *action,
1837                                     const uint64_t item_flags,
1838                                     struct rte_flow_error *error)
1839 {
1840         int ret = 0;
1841
1842         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1843         if (!ret) {
1844                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
1845                         return rte_flow_error_set(error, EINVAL,
1846                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1847                                                   NULL,
1848                                                   "no ipv6 item in pattern");
1849         }
1850         return ret;
1851 }
1852
1853 /**
1854  * Validate the modify-header TP actions.
1855  *
1856  * @param[in] action_flags
1857  *   Holds the actions detected until now.
1858  * @param[in] action
1859  *   Pointer to the modify action.
1860  * @param[in] item_flags
1861  *   Holds the items detected.
1862  * @param[out] error
1863  *   Pointer to error structure.
1864  *
1865  * @return
1866  *   0 on success, a negative errno value otherwise and rte_errno is set.
1867  */
1868 static int
1869 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
1870                                   const struct rte_flow_action *action,
1871                                   const uint64_t item_flags,
1872                                   struct rte_flow_error *error)
1873 {
1874         int ret = 0;
1875
1876         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1877         if (!ret) {
1878                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
1879                         return rte_flow_error_set(error, EINVAL,
1880                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1881                                                   NULL, "no transport layer "
1882                                                   "in pattern");
1883         }
1884         return ret;
1885 }
1886
1887 /**
1888  * Validate the modify-header actions of increment/decrement
1889  * TCP Sequence-number.
1890  *
1891  * @param[in] action_flags
1892  *   Holds the actions detected until now.
1893  * @param[in] action
1894  *   Pointer to the modify action.
1895  * @param[in] item_flags
1896  *   Holds the items detected.
1897  * @param[out] error
1898  *   Pointer to error structure.
1899  *
1900  * @return
1901  *   0 on success, a negative errno value otherwise and rte_errno is set.
1902  */
1903 static int
1904 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
1905                                        const struct rte_flow_action *action,
1906                                        const uint64_t item_flags,
1907                                        struct rte_flow_error *error)
1908 {
1909         int ret = 0;
1910
1911         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1912         if (!ret) {
1913                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1914                         return rte_flow_error_set(error, EINVAL,
1915                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1916                                                   NULL, "no TCP item in"
1917                                                   " pattern");
1918                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
1919                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
1920                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
1921                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
1922                         return rte_flow_error_set(error, EINVAL,
1923                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1924                                                   NULL,
1925                                                   "cannot decrease and increase"
1926                                                   " TCP sequence number"
1927                                                   " at the same time");
1928         }
1929         return ret;
1930 }
1931
1932 /**
1933  * Validate the modify-header actions of increment/decrement
1934  * TCP Acknowledgment number.
1935  *
1936  * @param[in] action_flags
1937  *   Holds the actions detected until now.
1938  * @param[in] action
1939  *   Pointer to the modify action.
1940  * @param[in] item_flags
1941  *   Holds the items detected.
1942  * @param[out] error
1943  *   Pointer to error structure.
1944  *
1945  * @return
1946  *   0 on success, a negative errno value otherwise and rte_errno is set.
1947  */
1948 static int
1949 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
1950                                        const struct rte_flow_action *action,
1951                                        const uint64_t item_flags,
1952                                        struct rte_flow_error *error)
1953 {
1954         int ret = 0;
1955
1956         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1957         if (!ret) {
1958                 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1959                         return rte_flow_error_set(error, EINVAL,
1960                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1961                                                   NULL, "no TCP item in"
1962                                                   " pattern");
1963                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
1964                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
1965                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
1966                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
1967                         return rte_flow_error_set(error, EINVAL,
1968                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1969                                                   NULL,
1970                                                   "cannot decrease and increase"
1971                                                   " TCP acknowledgment number"
1972                                                   " at the same time");
1973         }
1974         return ret;
1975 }
1976
1977 /**
1978  * Validate the modify-header TTL actions.
1979  *
1980  * @param[in] action_flags
1981  *   Holds the actions detected until now.
1982  * @param[in] action
1983  *   Pointer to the modify action.
1984  * @param[in] item_flags
1985  *   Holds the items detected.
1986  * @param[out] error
1987  *   Pointer to error structure.
1988  *
1989  * @return
1990  *   0 on success, a negative errno value otherwise and rte_errno is set.
1991  */
1992 static int
1993 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
1994                                    const struct rte_flow_action *action,
1995                                    const uint64_t item_flags,
1996                                    struct rte_flow_error *error)
1997 {
1998         int ret = 0;
1999
2000         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2001         if (!ret) {
2002                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
2003                         return rte_flow_error_set(error, EINVAL,
2004                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2005                                                   NULL,
2006                                                   "no IP protocol in pattern");
2007         }
2008         return ret;
2009 }
2010
2011 /**
2012  * Validate jump action.
2013  *
2014  * @param[in] action
2015  *   Pointer to the jump action.
2016  * @param[in] action_flags
2017  *   Holds the actions detected until now.
2018  * @param[in] group
2019  *   The group of the current flow.
2020  * @param[out] error
2021  *   Pointer to error structure.
2022  *
2023  * @return
2024  *   0 on success, a negative errno value otherwise and rte_errno is set.
2025  */
2026 static int
2027 flow_dv_validate_action_jump(const struct rte_flow_action *action,
2028                              uint64_t action_flags,
2029                              uint32_t group,
2030                              struct rte_flow_error *error)
2031 {
2032         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2033                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2034                 return rte_flow_error_set(error, EINVAL,
2035                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2036                                           "can't have 2 fate actions in"
2037                                           " same flow");
2038         if (!action->conf)
2039                 return rte_flow_error_set(error, EINVAL,
2040                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2041                                           NULL, "action configuration not set");
2042         if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
2043                 return rte_flow_error_set(error, EINVAL,
2044                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2045                                           "target group must be higher then"
2046                                           " the current flow group");
2047         return 0;
2048 }
2049
2050 /*
2051  * Validate the port_id action.
2052  *
2053  * @param[in] dev
2054  *   Pointer to rte_eth_dev structure.
2055  * @param[in] action_flags
2056  *   Bit-fields that holds the actions detected until now.
2057  * @param[in] action
2058  *   Port_id RTE action structure.
2059  * @param[in] attr
2060  *   Attributes of flow that includes this action.
2061  * @param[out] error
2062  *   Pointer to error structure.
2063  *
2064  * @return
2065  *   0 on success, a negative errno value otherwise and rte_errno is set.
2066  */
2067 static int
2068 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2069                                 uint64_t action_flags,
2070                                 const struct rte_flow_action *action,
2071                                 const struct rte_flow_attr *attr,
2072                                 struct rte_flow_error *error)
2073 {
2074         const struct rte_flow_action_port_id *port_id;
2075         uint16_t port;
2076         uint16_t esw_domain_id;
2077         uint16_t act_port_domain_id;
2078         int ret;
2079
2080         if (!attr->transfer)
2081                 return rte_flow_error_set(error, ENOTSUP,
2082                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2083                                           NULL,
2084                                           "port id action is valid in transfer"
2085                                           " mode only");
2086         if (!action || !action->conf)
2087                 return rte_flow_error_set(error, ENOTSUP,
2088                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2089                                           NULL,
2090                                           "port id action parameters must be"
2091                                           " specified");
2092         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2093                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2094                 return rte_flow_error_set(error, EINVAL,
2095                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2096                                           "can have only one fate actions in"
2097                                           " a flow");
2098         ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2099                                         &esw_domain_id, NULL);
2100         if (ret < 0)
2101                 return rte_flow_error_set(error, -ret,
2102                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2103                                           NULL,
2104                                           "failed to obtain E-Switch info");
2105         port_id = action->conf;
2106         port = port_id->original ? dev->data->port_id : port_id->id;
2107         ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2108         if (ret)
2109                 return rte_flow_error_set
2110                                 (error, -ret,
2111                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2112                                  "failed to obtain E-Switch port id for port");
2113         if (act_port_domain_id != esw_domain_id)
2114                 return rte_flow_error_set
2115                                 (error, -ret,
2116                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2117                                  "port does not belong to"
2118                                  " E-Switch being configured");
2119         return 0;
2120 }
2121
2122 /**
2123  * Find existing modify-header resource or create and register a new one.
2124  *
2125  * @param dev[in, out]
2126  *   Pointer to rte_eth_dev structure.
2127  * @param[in, out] resource
2128  *   Pointer to modify-header resource.
2129  * @parm[in, out] dev_flow
2130  *   Pointer to the dev_flow.
2131  * @param[out] error
2132  *   pointer to error structure.
2133  *
2134  * @return
2135  *   0 on success otherwise -errno and errno is set.
2136  */
2137 static int
2138 flow_dv_modify_hdr_resource_register
2139                         (struct rte_eth_dev *dev,
2140                          struct mlx5_flow_dv_modify_hdr_resource *resource,
2141                          struct mlx5_flow *dev_flow,
2142                          struct rte_flow_error *error)
2143 {
2144         struct mlx5_priv *priv = dev->data->dev_private;
2145         struct mlx5_ibv_shared *sh = priv->sh;
2146         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2147         struct mlx5dv_dr_domain *ns;
2148
2149         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2150                 ns = sh->fdb_domain;
2151         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2152                 ns = sh->tx_domain;
2153         else
2154                 ns = sh->rx_domain;
2155         resource->flags =
2156                 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2157         /* Lookup a matching resource from cache. */
2158         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2159                 if (resource->ft_type == cache_resource->ft_type &&
2160                     resource->actions_num == cache_resource->actions_num &&
2161                     resource->flags == cache_resource->flags &&
2162                     !memcmp((const void *)resource->actions,
2163                             (const void *)cache_resource->actions,
2164                             (resource->actions_num *
2165                                             sizeof(resource->actions[0])))) {
2166                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2167                                 (void *)cache_resource,
2168                                 rte_atomic32_read(&cache_resource->refcnt));
2169                         rte_atomic32_inc(&cache_resource->refcnt);
2170                         dev_flow->dv.modify_hdr = cache_resource;
2171                         return 0;
2172                 }
2173         }
2174         /* Register new modify-header resource. */
2175         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2176         if (!cache_resource)
2177                 return rte_flow_error_set(error, ENOMEM,
2178                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2179                                           "cannot allocate resource memory");
2180         *cache_resource = *resource;
2181         cache_resource->verbs_action =
2182                 mlx5_glue->dv_create_flow_action_modify_header
2183                                         (sh->ctx, cache_resource->ft_type,
2184                                          ns, cache_resource->flags,
2185                                          cache_resource->actions_num *
2186                                          sizeof(cache_resource->actions[0]),
2187                                          (uint64_t *)cache_resource->actions);
2188         if (!cache_resource->verbs_action) {
2189                 rte_free(cache_resource);
2190                 return rte_flow_error_set(error, ENOMEM,
2191                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2192                                           NULL, "cannot create action");
2193         }
2194         rte_atomic32_init(&cache_resource->refcnt);
2195         rte_atomic32_inc(&cache_resource->refcnt);
2196         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2197         dev_flow->dv.modify_hdr = cache_resource;
2198         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2199                 (void *)cache_resource,
2200                 rte_atomic32_read(&cache_resource->refcnt));
2201         return 0;
2202 }
2203
2204 #define MLX5_CNT_CONTAINER_RESIZE 64
2205
2206 /**
2207  * Get or create a flow counter.
2208  *
2209  * @param[in] dev
2210  *   Pointer to the Ethernet device structure.
2211  * @param[in] shared
2212  *   Indicate if this counter is shared with other flows.
2213  * @param[in] id
2214  *   Counter identifier.
2215  *
2216  * @return
2217  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2218  */
2219 static struct mlx5_flow_counter *
2220 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2221                                uint32_t id)
2222 {
2223         struct mlx5_priv *priv = dev->data->dev_private;
2224         struct mlx5_flow_counter *cnt = NULL;
2225         struct mlx5_devx_obj *dcs = NULL;
2226
2227         if (!priv->config.devx) {
2228                 rte_errno = ENOTSUP;
2229                 return NULL;
2230         }
2231         if (shared) {
2232                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2233                         if (cnt->shared && cnt->id == id) {
2234                                 cnt->ref_cnt++;
2235                                 return cnt;
2236                         }
2237                 }
2238         }
2239         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2240         if (!dcs)
2241                 return NULL;
2242         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2243         if (!cnt) {
2244                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2245                 rte_errno = ENOMEM;
2246                 return NULL;
2247         }
2248         struct mlx5_flow_counter tmpl = {
2249                 .shared = shared,
2250                 .ref_cnt = 1,
2251                 .id = id,
2252                 .dcs = dcs,
2253         };
2254         tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2255         if (!tmpl.action) {
2256                 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2257                 rte_errno = errno;
2258                 rte_free(cnt);
2259                 return NULL;
2260         }
2261         *cnt = tmpl;
2262         TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2263         return cnt;
2264 }
2265
2266 /**
2267  * Release a flow counter.
2268  *
2269  * @param[in] dev
2270  *   Pointer to the Ethernet device structure.
2271  * @param[in] counter
2272  *   Pointer to the counter handler.
2273  */
2274 static void
2275 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2276                                  struct mlx5_flow_counter *counter)
2277 {
2278         struct mlx5_priv *priv = dev->data->dev_private;
2279
2280         if (!counter)
2281                 return;
2282         if (--counter->ref_cnt == 0) {
2283                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2284                 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2285                 rte_free(counter);
2286         }
2287 }
2288
2289 /**
2290  * Query a devx flow counter.
2291  *
2292  * @param[in] dev
2293  *   Pointer to the Ethernet device structure.
2294  * @param[in] cnt
2295  *   Pointer to the flow counter.
2296  * @param[out] pkts
2297  *   The statistics value of packets.
2298  * @param[out] bytes
2299  *   The statistics value of bytes.
2300  *
2301  * @return
2302  *   0 on success, otherwise a negative errno value and rte_errno is set.
2303  */
2304 static inline int
2305 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2306                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2307                      uint64_t *bytes)
2308 {
2309         return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2310                                                 0, NULL, NULL, 0);
2311 }
2312
2313 /**
2314  * Get a pool by a counter.
2315  *
2316  * @param[in] cnt
2317  *   Pointer to the counter.
2318  *
2319  * @return
2320  *   The counter pool.
2321  */
2322 static struct mlx5_flow_counter_pool *
2323 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2324 {
2325         if (!cnt->batch) {
2326                 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2327                 return (struct mlx5_flow_counter_pool *)cnt - 1;
2328         }
2329         return cnt->pool;
2330 }
2331
2332 /**
2333  * Get a pool by devx counter ID.
2334  *
2335  * @param[in] cont
2336  *   Pointer to the counter container.
2337  * @param[in] id
2338  *   The counter devx ID.
2339  *
2340  * @return
2341  *   The counter pool pointer if exists, NULL otherwise,
2342  */
2343 static struct mlx5_flow_counter_pool *
2344 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2345 {
2346         struct mlx5_flow_counter_pool *pool;
2347
2348         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2349                 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2350                                 MLX5_COUNTERS_PER_POOL;
2351
2352                 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2353                         return pool;
2354         };
2355         return NULL;
2356 }
2357
2358 /**
2359  * Allocate a new memory for the counter values wrapped by all the needed
2360  * management.
2361  *
2362  * @param[in] dev
2363  *   Pointer to the Ethernet device structure.
2364  * @param[in] raws_n
2365  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2366  *
2367  * @return
2368  *   The new memory management pointer on success, otherwise NULL and rte_errno
2369  *   is set.
2370  */
2371 static struct mlx5_counter_stats_mem_mng *
2372 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2373 {
2374         struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2375                                         (dev->data->dev_private))->sh;
2376         struct mlx5_devx_mkey_attr mkey_attr;
2377         struct mlx5_counter_stats_mem_mng *mem_mng;
2378         volatile struct flow_counter_stats *raw_data;
2379         int size = (sizeof(struct flow_counter_stats) *
2380                         MLX5_COUNTERS_PER_POOL +
2381                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2382                         sizeof(struct mlx5_counter_stats_mem_mng);
2383         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2384         int i;
2385
2386         if (!mem) {
2387                 rte_errno = ENOMEM;
2388                 return NULL;
2389         }
2390         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2391         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2392         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2393                                                  IBV_ACCESS_LOCAL_WRITE);
2394         if (!mem_mng->umem) {
2395                 rte_errno = errno;
2396                 rte_free(mem);
2397                 return NULL;
2398         }
2399         mkey_attr.addr = (uintptr_t)mem;
2400         mkey_attr.size = size;
2401         mkey_attr.umem_id = mem_mng->umem->umem_id;
2402         mkey_attr.pd = sh->pdn;
2403         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2404         if (!mem_mng->dm) {
2405                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
2406                 rte_errno = errno;
2407                 rte_free(mem);
2408                 return NULL;
2409         }
2410         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
2411         raw_data = (volatile struct flow_counter_stats *)mem;
2412         for (i = 0; i < raws_n; ++i) {
2413                 mem_mng->raws[i].mem_mng = mem_mng;
2414                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
2415         }
2416         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
2417         return mem_mng;
2418 }
2419
2420 /**
2421  * Resize a counter container.
2422  *
2423  * @param[in] dev
2424  *   Pointer to the Ethernet device structure.
2425  * @param[in] batch
2426  *   Whether the pool is for counter that was allocated by batch command.
2427  *
2428  * @return
2429  *   The new container pointer on success, otherwise NULL and rte_errno is set.
2430  */
2431 static struct mlx5_pools_container *
2432 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
2433 {
2434         struct mlx5_priv *priv = dev->data->dev_private;
2435         struct mlx5_pools_container *cont =
2436                         MLX5_CNT_CONTAINER(priv->sh, batch, 0);
2437         struct mlx5_pools_container *new_cont =
2438                         MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
2439         struct mlx5_counter_stats_mem_mng *mem_mng;
2440         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
2441         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
2442         int i;
2443
2444         if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
2445                 /* The last resize still hasn't detected by the host thread. */
2446                 rte_errno = EAGAIN;
2447                 return NULL;
2448         }
2449         new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
2450         if (!new_cont->pools) {
2451                 rte_errno = ENOMEM;
2452                 return NULL;
2453         }
2454         if (cont->n)
2455                 memcpy(new_cont->pools, cont->pools, cont->n *
2456                        sizeof(struct mlx5_flow_counter_pool *));
2457         mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
2458                 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
2459         if (!mem_mng) {
2460                 rte_free(new_cont->pools);
2461                 return NULL;
2462         }
2463         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
2464                 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
2465                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
2466                                  i, next);
2467         new_cont->n = resize;
2468         rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
2469         TAILQ_INIT(&new_cont->pool_list);
2470         TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
2471         new_cont->init_mem_mng = mem_mng;
2472         rte_cio_wmb();
2473          /* Flip the master container. */
2474         priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
2475         return new_cont;
2476 }
2477
2478 /**
2479  * Query a devx flow counter.
2480  *
2481  * @param[in] dev
2482  *   Pointer to the Ethernet device structure.
2483  * @param[in] cnt
2484  *   Pointer to the flow counter.
2485  * @param[out] pkts
2486  *   The statistics value of packets.
2487  * @param[out] bytes
2488  *   The statistics value of bytes.
2489  *
2490  * @return
2491  *   0 on success, otherwise a negative errno value and rte_errno is set.
2492  */
2493 static inline int
2494 _flow_dv_query_count(struct rte_eth_dev *dev,
2495                      struct mlx5_flow_counter *cnt, uint64_t *pkts,
2496                      uint64_t *bytes)
2497 {
2498         struct mlx5_priv *priv = dev->data->dev_private;
2499         struct mlx5_flow_counter_pool *pool =
2500                         flow_dv_counter_pool_get(cnt);
2501         int offset = cnt - &pool->counters_raw[0];
2502
2503         if (priv->counter_fallback)
2504                 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
2505
2506         rte_spinlock_lock(&pool->sl);
2507         /*
2508          * The single counters allocation may allocate smaller ID than the
2509          * current allocated in parallel to the host reading.
2510          * In this case the new counter values must be reported as 0.
2511          */
2512         if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
2513                 *pkts = 0;
2514                 *bytes = 0;
2515         } else {
2516                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
2517                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
2518         }
2519         rte_spinlock_unlock(&pool->sl);
2520         return 0;
2521 }
2522
2523 /**
2524  * Create and initialize a new counter pool.
2525  *
2526  * @param[in] dev
2527  *   Pointer to the Ethernet device structure.
2528  * @param[out] dcs
2529  *   The devX counter handle.
2530  * @param[in] batch
2531  *   Whether the pool is for counter that was allocated by batch command.
2532  *
2533  * @return
2534  *   A new pool pointer on success, NULL otherwise and rte_errno is set.
2535  */
2536 static struct mlx5_flow_counter_pool *
2537 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
2538                     uint32_t batch)
2539 {
2540         struct mlx5_priv *priv = dev->data->dev_private;
2541         struct mlx5_flow_counter_pool *pool;
2542         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2543                                                                0);
2544         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
2545         uint32_t size;
2546
2547         if (cont->n == n_valid) {
2548                 cont = flow_dv_container_resize(dev, batch);
2549                 if (!cont)
2550                         return NULL;
2551         }
2552         size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
2553                         sizeof(struct mlx5_flow_counter);
2554         pool = rte_calloc(__func__, 1, size, 0);
2555         if (!pool) {
2556                 rte_errno = ENOMEM;
2557                 return NULL;
2558         }
2559         pool->min_dcs = dcs;
2560         pool->raw = cont->init_mem_mng->raws + n_valid %
2561                                                      MLX5_CNT_CONTAINER_RESIZE;
2562         pool->raw_hw = NULL;
2563         rte_spinlock_init(&pool->sl);
2564         /*
2565          * The generation of the new allocated counters in this pool is 0, 2 in
2566          * the pool generation makes all the counters valid for allocation.
2567          */
2568         rte_atomic64_set(&pool->query_gen, 0x2);
2569         TAILQ_INIT(&pool->counters);
2570         TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2571         cont->pools[n_valid] = pool;
2572         /* Pool initialization must be updated before host thread access. */
2573         rte_cio_wmb();
2574         rte_atomic16_add(&cont->n_valid, 1);
2575         return pool;
2576 }
2577
2578 /**
2579  * Prepare a new counter and/or a new counter pool.
2580  *
2581  * @param[in] dev
2582  *   Pointer to the Ethernet device structure.
2583  * @param[out] cnt_free
2584  *   Where to put the pointer of a new counter.
2585  * @param[in] batch
2586  *   Whether the pool is for counter that was allocated by batch command.
2587  *
2588  * @return
2589  *   The free counter pool pointer and @p cnt_free is set on success,
2590  *   NULL otherwise and rte_errno is set.
2591  */
2592 static struct mlx5_flow_counter_pool *
2593 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
2594                              struct mlx5_flow_counter **cnt_free,
2595                              uint32_t batch)
2596 {
2597         struct mlx5_priv *priv = dev->data->dev_private;
2598         struct mlx5_flow_counter_pool *pool;
2599         struct mlx5_devx_obj *dcs = NULL;
2600         struct mlx5_flow_counter *cnt;
2601         uint32_t i;
2602
2603         if (!batch) {
2604                 /* bulk_bitmap must be 0 for single counter allocation. */
2605                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2606                 if (!dcs)
2607                         return NULL;
2608                 pool = flow_dv_find_pool_by_id
2609                         (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
2610                 if (!pool) {
2611                         pool = flow_dv_pool_create(dev, dcs, batch);
2612                         if (!pool) {
2613                                 mlx5_devx_cmd_destroy(dcs);
2614                                 return NULL;
2615                         }
2616                 } else if (dcs->id < pool->min_dcs->id) {
2617                         rte_atomic64_set(&pool->a64_dcs,
2618                                          (int64_t)(uintptr_t)dcs);
2619                 }
2620                 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
2621                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2622                 cnt->dcs = dcs;
2623                 *cnt_free = cnt;
2624                 return pool;
2625         }
2626         /* bulk_bitmap is in 128 counters units. */
2627         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
2628                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
2629         if (!dcs) {
2630                 rte_errno = ENODATA;
2631                 return NULL;
2632         }
2633         pool = flow_dv_pool_create(dev, dcs, batch);
2634         if (!pool) {
2635                 mlx5_devx_cmd_destroy(dcs);
2636                 return NULL;
2637         }
2638         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2639                 cnt = &pool->counters_raw[i];
2640                 cnt->pool = pool;
2641                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2642         }
2643         *cnt_free = &pool->counters_raw[0];
2644         return pool;
2645 }
2646
2647 /**
2648  * Search for existed shared counter.
2649  *
2650  * @param[in] cont
2651  *   Pointer to the relevant counter pool container.
2652  * @param[in] id
2653  *   The shared counter ID to search.
2654  *
2655  * @return
2656  *   NULL if not existed, otherwise pointer to the shared counter.
2657  */
2658 static struct mlx5_flow_counter *
2659 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
2660                               uint32_t id)
2661 {
2662         static struct mlx5_flow_counter *cnt;
2663         struct mlx5_flow_counter_pool *pool;
2664         int i;
2665
2666         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2667                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2668                         cnt = &pool->counters_raw[i];
2669                         if (cnt->ref_cnt && cnt->shared && cnt->id == id)
2670                                 return cnt;
2671                 }
2672         }
2673         return NULL;
2674 }
2675
2676 /**
2677  * Allocate a flow counter.
2678  *
2679  * @param[in] dev
2680  *   Pointer to the Ethernet device structure.
2681  * @param[in] shared
2682  *   Indicate if this counter is shared with other flows.
2683  * @param[in] id
2684  *   Counter identifier.
2685  * @param[in] group
2686  *   Counter flow group.
2687  *
2688  * @return
2689  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
2690  */
2691 static struct mlx5_flow_counter *
2692 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
2693                       uint16_t group)
2694 {
2695         struct mlx5_priv *priv = dev->data->dev_private;
2696         struct mlx5_flow_counter_pool *pool = NULL;
2697         struct mlx5_flow_counter *cnt_free = NULL;
2698         /*
2699          * Currently group 0 flow counter cannot be assigned to a flow if it is
2700          * not the first one in the batch counter allocation, so it is better
2701          * to allocate counters one by one for these flows in a separate
2702          * container.
2703          * A counter can be shared between different groups so need to take
2704          * shared counters from the single container.
2705          */
2706         uint32_t batch = (group && !shared) ? 1 : 0;
2707         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2708                                                                0);
2709
2710         if (priv->counter_fallback)
2711                 return flow_dv_counter_alloc_fallback(dev, shared, id);
2712         if (!priv->config.devx) {
2713                 rte_errno = ENOTSUP;
2714                 return NULL;
2715         }
2716         if (shared) {
2717                 cnt_free = flow_dv_counter_shared_search(cont, id);
2718                 if (cnt_free) {
2719                         if (cnt_free->ref_cnt + 1 == 0) {
2720                                 rte_errno = E2BIG;
2721                                 return NULL;
2722                         }
2723                         cnt_free->ref_cnt++;
2724                         return cnt_free;
2725                 }
2726         }
2727         /* Pools which has a free counters are in the start. */
2728         TAILQ_FOREACH(pool, &cont->pool_list, next) {
2729                 /*
2730                  * The free counter reset values must be updated between the
2731                  * counter release to the counter allocation, so, at least one
2732                  * query must be done in this time. ensure it by saving the
2733                  * query generation in the release time.
2734                  * The free list is sorted according to the generation - so if
2735                  * the first one is not updated, all the others are not
2736                  * updated too.
2737                  */
2738                 cnt_free = TAILQ_FIRST(&pool->counters);
2739                 if (cnt_free && cnt_free->query_gen + 1 <
2740                     rte_atomic64_read(&pool->query_gen))
2741                         break;
2742                 cnt_free = NULL;
2743         }
2744         if (!cnt_free) {
2745                 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
2746                 if (!pool)
2747                         return NULL;
2748         }
2749         cnt_free->batch = batch;
2750         /* Create a DV counter action only in the first time usage. */
2751         if (!cnt_free->action) {
2752                 uint16_t offset;
2753                 struct mlx5_devx_obj *dcs;
2754
2755                 if (batch) {
2756                         offset = cnt_free - &pool->counters_raw[0];
2757                         dcs = pool->min_dcs;
2758                 } else {
2759                         offset = 0;
2760                         dcs = cnt_free->dcs;
2761                 }
2762                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
2763                                         (dcs->obj, offset);
2764                 if (!cnt_free->action) {
2765                         rte_errno = errno;
2766                         return NULL;
2767                 }
2768         }
2769         /* Update the counter reset values. */
2770         if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
2771                                  &cnt_free->bytes))
2772                 return NULL;
2773         cnt_free->shared = shared;
2774         cnt_free->ref_cnt = 1;
2775         cnt_free->id = id;
2776         if (!priv->sh->cmng.query_thread_on)
2777                 /* Start the asynchronous batch query by the host thread. */
2778                 mlx5_set_query_alarm(priv->sh);
2779         TAILQ_REMOVE(&pool->counters, cnt_free, next);
2780         if (TAILQ_EMPTY(&pool->counters)) {
2781                 /* Move the pool to the end of the container pool list. */
2782                 TAILQ_REMOVE(&cont->pool_list, pool, next);
2783                 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2784         }
2785         return cnt_free;
2786 }
2787
2788 /**
2789  * Release a flow counter.
2790  *
2791  * @param[in] dev
2792  *   Pointer to the Ethernet device structure.
2793  * @param[in] counter
2794  *   Pointer to the counter handler.
2795  */
2796 static void
2797 flow_dv_counter_release(struct rte_eth_dev *dev,
2798                         struct mlx5_flow_counter *counter)
2799 {
2800         struct mlx5_priv *priv = dev->data->dev_private;
2801
2802         if (!counter)
2803                 return;
2804         if (priv->counter_fallback) {
2805                 flow_dv_counter_release_fallback(dev, counter);
2806                 return;
2807         }
2808         if (--counter->ref_cnt == 0) {
2809                 struct mlx5_flow_counter_pool *pool =
2810                                 flow_dv_counter_pool_get(counter);
2811
2812                 /* Put the counter in the end - the last updated one. */
2813                 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
2814                 counter->query_gen = rte_atomic64_read(&pool->query_gen);
2815         }
2816 }
2817
2818 /**
2819  * Verify the @p attributes will be correctly understood by the NIC and store
2820  * them in the @p flow if everything is correct.
2821  *
2822  * @param[in] dev
2823  *   Pointer to dev struct.
2824  * @param[in] attributes
2825  *   Pointer to flow attributes
2826  * @param[out] error
2827  *   Pointer to error structure.
2828  *
2829  * @return
2830  *   0 on success, a negative errno value otherwise and rte_errno is set.
2831  */
2832 static int
2833 flow_dv_validate_attributes(struct rte_eth_dev *dev,
2834                             const struct rte_flow_attr *attributes,
2835                             struct rte_flow_error *error)
2836 {
2837         struct mlx5_priv *priv = dev->data->dev_private;
2838         uint32_t priority_max = priv->config.flow_prio - 1;
2839
2840 #ifndef HAVE_MLX5DV_DR
2841         if (attributes->group)
2842                 return rte_flow_error_set(error, ENOTSUP,
2843                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2844                                           NULL,
2845                                           "groups is not supported");
2846 #endif
2847         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
2848             attributes->priority >= priority_max)
2849                 return rte_flow_error_set(error, ENOTSUP,
2850                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2851                                           NULL,
2852                                           "priority out of range");
2853         if (attributes->transfer) {
2854                 if (!priv->config.dv_esw_en)
2855                         return rte_flow_error_set
2856                                 (error, ENOTSUP,
2857                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2858                                  "E-Switch dr is not supported");
2859                 if (!(priv->representor || priv->master))
2860                         return rte_flow_error_set
2861                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2862                                  NULL, "E-Switch configuration can only be"
2863                                  " done by a master or a representor device");
2864                 if (attributes->egress)
2865                         return rte_flow_error_set
2866                                 (error, ENOTSUP,
2867                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
2868                                  "egress is not supported");
2869                 if (attributes->group >= MLX5_MAX_TABLES_FDB)
2870                         return rte_flow_error_set
2871                                 (error, EINVAL,
2872                                  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2873                                  NULL, "group must be smaller than "
2874                                  RTE_STR(MLX5_MAX_TABLES_FDB));
2875         }
2876         if (!(attributes->egress ^ attributes->ingress))
2877                 return rte_flow_error_set(error, ENOTSUP,
2878                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
2879                                           "must specify exactly one of "
2880                                           "ingress or egress");
2881         return 0;
2882 }
2883
2884 /**
2885  * Internal validation function. For validating both actions and items.
2886  *
2887  * @param[in] dev
2888  *   Pointer to the rte_eth_dev structure.
2889  * @param[in] attr
2890  *   Pointer to the flow attributes.
2891  * @param[in] items
2892  *   Pointer to the list of items.
2893  * @param[in] actions
2894  *   Pointer to the list of actions.
2895  * @param[out] error
2896  *   Pointer to the error structure.
2897  *
2898  * @return
2899  *   0 on success, a negative errno value otherwise and rte_errno is set.
2900  */
2901 static int
2902 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
2903                  const struct rte_flow_item items[],
2904                  const struct rte_flow_action actions[],
2905                  struct rte_flow_error *error)
2906 {
2907         int ret;
2908         uint64_t action_flags = 0;
2909         uint64_t item_flags = 0;
2910         uint64_t last_item = 0;
2911         uint8_t next_protocol = 0xff;
2912         int actions_n = 0;
2913         const struct rte_flow_item *gre_item = NULL;
2914         struct rte_flow_item_tcp nic_tcp_mask = {
2915                 .hdr = {
2916                         .tcp_flags = 0xFF,
2917                         .src_port = RTE_BE16(UINT16_MAX),
2918                         .dst_port = RTE_BE16(UINT16_MAX),
2919                 }
2920         };
2921
2922         if (items == NULL)
2923                 return -1;
2924         ret = flow_dv_validate_attributes(dev, attr, error);
2925         if (ret < 0)
2926                 return ret;
2927         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2928                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2929                 switch (items->type) {
2930                 case RTE_FLOW_ITEM_TYPE_VOID:
2931                         break;
2932                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2933                         ret = flow_dv_validate_item_port_id
2934                                         (dev, items, attr, item_flags, error);
2935                         if (ret < 0)
2936                                 return ret;
2937                         last_item = MLX5_FLOW_ITEM_PORT_ID;
2938                         break;
2939                 case RTE_FLOW_ITEM_TYPE_ETH:
2940                         ret = mlx5_flow_validate_item_eth(items, item_flags,
2941                                                           error);
2942                         if (ret < 0)
2943                                 return ret;
2944                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2945                                              MLX5_FLOW_LAYER_OUTER_L2;
2946                         break;
2947                 case RTE_FLOW_ITEM_TYPE_VLAN:
2948                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
2949                                                            dev, error);
2950                         if (ret < 0)
2951                                 return ret;
2952                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2953                                              MLX5_FLOW_LAYER_OUTER_VLAN;
2954                         break;
2955                 case RTE_FLOW_ITEM_TYPE_IPV4:
2956                         mlx5_flow_tunnel_ip_check(items, next_protocol,
2957                                                   &item_flags, &tunnel);
2958                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2959                                                            NULL, error);
2960                         if (ret < 0)
2961                                 return ret;
2962                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2963                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2964                         if (items->mask != NULL &&
2965                             ((const struct rte_flow_item_ipv4 *)
2966                              items->mask)->hdr.next_proto_id) {
2967                                 next_protocol =
2968                                         ((const struct rte_flow_item_ipv4 *)
2969                                          (items->spec))->hdr.next_proto_id;
2970                                 next_protocol &=
2971                                         ((const struct rte_flow_item_ipv4 *)
2972                                          (items->mask))->hdr.next_proto_id;
2973                         } else {
2974                                 /* Reset for inner layer. */
2975                                 next_protocol = 0xff;
2976                         }
2977                         break;
2978                 case RTE_FLOW_ITEM_TYPE_IPV6:
2979                         mlx5_flow_tunnel_ip_check(items, next_protocol,
2980                                                   &item_flags, &tunnel);
2981                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2982                                                            NULL, error);
2983                         if (ret < 0)
2984                                 return ret;
2985                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2986                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2987                         if (items->mask != NULL &&
2988                             ((const struct rte_flow_item_ipv6 *)
2989                              items->mask)->hdr.proto) {
2990                                 next_protocol =
2991                                         ((const struct rte_flow_item_ipv6 *)
2992                                          items->spec)->hdr.proto;
2993                                 next_protocol &=
2994                                         ((const struct rte_flow_item_ipv6 *)
2995                                          items->mask)->hdr.proto;
2996                         } else {
2997                                 /* Reset for inner layer. */
2998                                 next_protocol = 0xff;
2999                         }
3000                         break;
3001                 case RTE_FLOW_ITEM_TYPE_TCP:
3002                         ret = mlx5_flow_validate_item_tcp
3003                                                 (items, item_flags,
3004                                                  next_protocol,
3005                                                  &nic_tcp_mask,
3006                                                  error);
3007                         if (ret < 0)
3008                                 return ret;
3009                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3010                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
3011                         break;
3012                 case RTE_FLOW_ITEM_TYPE_UDP:
3013                         ret = mlx5_flow_validate_item_udp(items, item_flags,
3014                                                           next_protocol,
3015                                                           error);
3016                         if (ret < 0)
3017                                 return ret;
3018                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3019                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
3020                         break;
3021                 case RTE_FLOW_ITEM_TYPE_GRE:
3022                         ret = mlx5_flow_validate_item_gre(items, item_flags,
3023                                                           next_protocol, error);
3024                         if (ret < 0)
3025                                 return ret;
3026                         gre_item = items;
3027                         last_item = MLX5_FLOW_LAYER_GRE;
3028                         break;
3029                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3030                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
3031                                                             next_protocol,
3032                                                             error);
3033                         if (ret < 0)
3034                                 return ret;
3035                         last_item = MLX5_FLOW_LAYER_NVGRE;
3036                         break;
3037                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
3038                         ret = mlx5_flow_validate_item_gre_key
3039                                 (items, item_flags, gre_item, error);
3040                         if (ret < 0)
3041                                 return ret;
3042                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
3043                         break;
3044                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3045                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
3046                                                             error);
3047                         if (ret < 0)
3048                                 return ret;
3049                         last_item = MLX5_FLOW_LAYER_VXLAN;
3050                         break;
3051                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3052                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
3053                                                                 item_flags, dev,
3054                                                                 error);
3055                         if (ret < 0)
3056                                 return ret;
3057                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3058                         break;
3059                 case RTE_FLOW_ITEM_TYPE_MPLS:
3060                         ret = mlx5_flow_validate_item_mpls(dev, items,
3061                                                            item_flags,
3062                                                            last_item, error);
3063                         if (ret < 0)
3064                                 return ret;
3065                         last_item = MLX5_FLOW_LAYER_MPLS;
3066                         break;
3067                 case RTE_FLOW_ITEM_TYPE_META:
3068                         ret = flow_dv_validate_item_meta(dev, items, attr,
3069                                                          error);
3070                         if (ret < 0)
3071                                 return ret;
3072                         last_item = MLX5_FLOW_ITEM_METADATA;
3073                         break;
3074                 case RTE_FLOW_ITEM_TYPE_ICMP:
3075                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
3076                                                            next_protocol,
3077                                                            error);
3078                         if (ret < 0)
3079                                 return ret;
3080                         last_item = MLX5_FLOW_LAYER_ICMP;
3081                         break;
3082                 case RTE_FLOW_ITEM_TYPE_ICMP6:
3083                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3084                                                             next_protocol,
3085                                                             error);
3086                         if (ret < 0)
3087                                 return ret;
3088                         last_item = MLX5_FLOW_LAYER_ICMP6;
3089                         break;
3090                 default:
3091                         return rte_flow_error_set(error, ENOTSUP,
3092                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3093                                                   NULL, "item not supported");
3094                 }
3095                 item_flags |= last_item;
3096         }
3097         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3098                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3099                         return rte_flow_error_set(error, ENOTSUP,
3100                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3101                                                   actions, "too many actions");
3102                 switch (actions->type) {
3103                 case RTE_FLOW_ACTION_TYPE_VOID:
3104                         break;
3105                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3106                         ret = flow_dv_validate_action_port_id(dev,
3107                                                               action_flags,
3108                                                               actions,
3109                                                               attr,
3110                                                               error);
3111                         if (ret)
3112                                 return ret;
3113                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3114                         ++actions_n;
3115                         break;
3116                 case RTE_FLOW_ACTION_TYPE_FLAG:
3117                         ret = mlx5_flow_validate_action_flag(action_flags,
3118                                                              attr, error);
3119                         if (ret < 0)
3120                                 return ret;
3121                         action_flags |= MLX5_FLOW_ACTION_FLAG;
3122                         ++actions_n;
3123                         break;
3124                 case RTE_FLOW_ACTION_TYPE_MARK:
3125                         ret = mlx5_flow_validate_action_mark(actions,
3126                                                              action_flags,
3127                                                              attr, error);
3128                         if (ret < 0)
3129                                 return ret;
3130                         action_flags |= MLX5_FLOW_ACTION_MARK;
3131                         ++actions_n;
3132                         break;
3133                 case RTE_FLOW_ACTION_TYPE_DROP:
3134                         ret = mlx5_flow_validate_action_drop(action_flags,
3135                                                              attr, error);
3136                         if (ret < 0)
3137                                 return ret;
3138                         action_flags |= MLX5_FLOW_ACTION_DROP;
3139                         ++actions_n;
3140                         break;
3141                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3142                         ret = mlx5_flow_validate_action_queue(actions,
3143                                                               action_flags, dev,
3144                                                               attr, error);
3145                         if (ret < 0)
3146                                 return ret;
3147                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
3148                         ++actions_n;
3149                         break;
3150                 case RTE_FLOW_ACTION_TYPE_RSS:
3151                         ret = mlx5_flow_validate_action_rss(actions,
3152                                                             action_flags, dev,
3153                                                             attr, item_flags,
3154                                                             error);
3155                         if (ret < 0)
3156                                 return ret;
3157                         action_flags |= MLX5_FLOW_ACTION_RSS;
3158                         ++actions_n;
3159                         break;
3160                 case RTE_FLOW_ACTION_TYPE_COUNT:
3161                         ret = flow_dv_validate_action_count(dev, error);
3162                         if (ret < 0)
3163                                 return ret;
3164                         action_flags |= MLX5_FLOW_ACTION_COUNT;
3165                         ++actions_n;
3166                         break;
3167                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3168                         if (flow_dv_validate_action_pop_vlan(dev,
3169                                                              action_flags,
3170                                                              actions,
3171                                                              item_flags, attr,
3172                                                              error))
3173                                 return -rte_errno;
3174                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
3175                         ++actions_n;
3176                         break;
3177                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3178                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3179                         ret = flow_dv_validate_action_l2_encap(action_flags,
3180                                                                actions, attr,
3181                                                                error);
3182                         if (ret < 0)
3183                                 return ret;
3184                         action_flags |= actions->type ==
3185                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3186                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
3187                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
3188                         ++actions_n;
3189                         break;
3190                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3191                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3192                         ret = flow_dv_validate_action_l2_decap(action_flags,
3193                                                                attr, error);
3194                         if (ret < 0)
3195                                 return ret;
3196                         action_flags |= actions->type ==
3197                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3198                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
3199                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
3200                         ++actions_n;
3201                         break;
3202                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3203                         ret = flow_dv_validate_action_raw_encap(action_flags,
3204                                                                 actions, attr,
3205                                                                 error);
3206                         if (ret < 0)
3207                                 return ret;
3208                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3209                         ++actions_n;
3210                         break;
3211                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3212                         ret = flow_dv_validate_action_raw_decap(action_flags,
3213                                                                 actions, attr,
3214                                                                 error);
3215                         if (ret < 0)
3216                                 return ret;
3217                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3218                         ++actions_n;
3219                         break;
3220                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3221                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3222                         ret = flow_dv_validate_action_modify_mac(action_flags,
3223                                                                  actions,
3224                                                                  item_flags,
3225                                                                  error);
3226                         if (ret < 0)
3227                                 return ret;
3228                         /* Count all modify-header actions as one action. */
3229                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3230                                 ++actions_n;
3231                         action_flags |= actions->type ==
3232                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3233                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
3234                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
3235                         break;
3236
3237                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3238                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3239                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
3240                                                                   actions,
3241                                                                   item_flags,
3242                                                                   error);
3243                         if (ret < 0)
3244                                 return ret;
3245                         /* Count all modify-header actions as one action. */
3246                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3247                                 ++actions_n;
3248                         action_flags |= actions->type ==
3249                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3250                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3251                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
3252                         break;
3253                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3254                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3255                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
3256                                                                   actions,
3257                                                                   item_flags,
3258                                                                   error);
3259                         if (ret < 0)
3260                                 return ret;
3261                         /* Count all modify-header actions as one action. */
3262                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3263                                 ++actions_n;
3264                         action_flags |= actions->type ==
3265                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3266                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3267                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
3268                         break;
3269                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3270                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3271                         ret = flow_dv_validate_action_modify_tp(action_flags,
3272                                                                 actions,
3273                                                                 item_flags,
3274                                                                 error);
3275                         if (ret < 0)
3276                                 return ret;
3277                         /* Count all modify-header actions as one action. */
3278                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3279                                 ++actions_n;
3280                         action_flags |= actions->type ==
3281                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3282                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
3283                                                 MLX5_FLOW_ACTION_SET_TP_DST;
3284                         break;
3285                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3286                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3287                         ret = flow_dv_validate_action_modify_ttl(action_flags,
3288                                                                  actions,
3289                                                                  item_flags,
3290                                                                  error);
3291                         if (ret < 0)
3292                                 return ret;
3293                         /* Count all modify-header actions as one action. */
3294                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3295                                 ++actions_n;
3296                         action_flags |= actions->type ==
3297                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
3298                                                 MLX5_FLOW_ACTION_SET_TTL :
3299                                                 MLX5_FLOW_ACTION_DEC_TTL;
3300                         break;
3301                 case RTE_FLOW_ACTION_TYPE_JUMP:
3302                         ret = flow_dv_validate_action_jump(actions,
3303                                                            action_flags,
3304                                                            attr->group, error);
3305                         if (ret)
3306                                 return ret;
3307                         ++actions_n;
3308                         action_flags |= MLX5_FLOW_ACTION_JUMP;
3309                         break;
3310                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3311                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3312                         ret = flow_dv_validate_action_modify_tcp_seq
3313                                                                 (action_flags,
3314                                                                  actions,
3315                                                                  item_flags,
3316                                                                  error);
3317                         if (ret < 0)
3318                                 return ret;
3319                         /* Count all modify-header actions as one action. */
3320                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3321                                 ++actions_n;
3322                         action_flags |= actions->type ==
3323                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
3324                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
3325                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
3326                         break;
3327                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3328                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3329                         ret = flow_dv_validate_action_modify_tcp_ack
3330                                                                 (action_flags,
3331                                                                  actions,
3332                                                                  item_flags,
3333                                                                  error);
3334                         if (ret < 0)
3335                                 return ret;
3336                         /* Count all modify-header actions as one action. */
3337                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3338                                 ++actions_n;
3339                         action_flags |= actions->type ==
3340                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
3341                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
3342                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
3343                         break;
3344                 default:
3345                         return rte_flow_error_set(error, ENOTSUP,
3346                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3347                                                   actions,
3348                                                   "action not supported");
3349                 }
3350         }
3351         if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3352             (action_flags & MLX5_FLOW_VLAN_ACTIONS))
3353                 return rte_flow_error_set(error, ENOTSUP,
3354                                           RTE_FLOW_ERROR_TYPE_ACTION,
3355                                           actions,
3356                                           "can't have vxlan and vlan"
3357                                           " actions in the same rule");
3358         /* Eswitch has few restrictions on using items and actions */
3359         if (attr->transfer) {
3360                 if (action_flags & MLX5_FLOW_ACTION_FLAG)
3361                         return rte_flow_error_set(error, ENOTSUP,
3362                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                   NULL,
3364                                                   "unsupported action FLAG");
3365                 if (action_flags & MLX5_FLOW_ACTION_MARK)
3366                         return rte_flow_error_set(error, ENOTSUP,
3367                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3368                                                   NULL,
3369                                                   "unsupported action MARK");
3370                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
3371                         return rte_flow_error_set(error, ENOTSUP,
3372                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3373                                                   NULL,
3374                                                   "unsupported action QUEUE");
3375                 if (action_flags & MLX5_FLOW_ACTION_RSS)
3376                         return rte_flow_error_set(error, ENOTSUP,
3377                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3378                                                   NULL,
3379                                                   "unsupported action RSS");
3380                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3381                         return rte_flow_error_set(error, EINVAL,
3382                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3383                                                   actions,
3384                                                   "no fate action is found");
3385         } else {
3386                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
3387                         return rte_flow_error_set(error, EINVAL,
3388                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3389                                                   actions,
3390                                                   "no fate action is found");
3391         }
3392         return 0;
3393 }
3394
3395 /**
3396  * Internal preparation function. Allocates the DV flow size,
3397  * this size is constant.
3398  *
3399  * @param[in] attr
3400  *   Pointer to the flow attributes.
3401  * @param[in] items
3402  *   Pointer to the list of items.
3403  * @param[in] actions
3404  *   Pointer to the list of actions.
3405  * @param[out] error
3406  *   Pointer to the error structure.
3407  *
3408  * @return
3409  *   Pointer to mlx5_flow object on success,
3410  *   otherwise NULL and rte_errno is set.
3411  */
3412 static struct mlx5_flow *
3413 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
3414                 const struct rte_flow_item items[] __rte_unused,
3415                 const struct rte_flow_action actions[] __rte_unused,
3416                 struct rte_flow_error *error)
3417 {
3418         uint32_t size = sizeof(struct mlx5_flow);
3419         struct mlx5_flow *flow;
3420
3421         flow = rte_calloc(__func__, 1, size, 0);
3422         if (!flow) {
3423                 rte_flow_error_set(error, ENOMEM,
3424                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3425                                    "not enough memory to create flow");
3426                 return NULL;
3427         }
3428         flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
3429         return flow;
3430 }
3431
3432 #ifndef NDEBUG
3433 /**
3434  * Sanity check for match mask and value. Similar to check_valid_spec() in
3435  * kernel driver. If unmasked bit is present in value, it returns failure.
3436  *
3437  * @param match_mask
3438  *   pointer to match mask buffer.
3439  * @param match_value
3440  *   pointer to match value buffer.
3441  *
3442  * @return
3443  *   0 if valid, -EINVAL otherwise.
3444  */
3445 static int
3446 flow_dv_check_valid_spec(void *match_mask, void *match_value)
3447 {
3448         uint8_t *m = match_mask;
3449         uint8_t *v = match_value;
3450         unsigned int i;
3451
3452         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
3453                 if (v[i] & ~m[i]) {
3454                         DRV_LOG(ERR,
3455                                 "match_value differs from match_criteria"
3456                                 " %p[%u] != %p[%u]",
3457                                 match_value, i, match_mask, i);
3458                         return -EINVAL;
3459                 }
3460         }
3461         return 0;
3462 }
3463 #endif
3464
3465 /**
3466  * Add Ethernet item to matcher and to the value.
3467  *
3468  * @param[in, out] matcher
3469  *   Flow matcher.
3470  * @param[in, out] key
3471  *   Flow matcher value.
3472  * @param[in] item
3473  *   Flow pattern to translate.
3474  * @param[in] inner
3475  *   Item is inner pattern.
3476  */
3477 static void
3478 flow_dv_translate_item_eth(void *matcher, void *key,
3479                            const struct rte_flow_item *item, int inner)
3480 {
3481         const struct rte_flow_item_eth *eth_m = item->mask;
3482         const struct rte_flow_item_eth *eth_v = item->spec;
3483         const struct rte_flow_item_eth nic_mask = {
3484                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3485                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3486                 .type = RTE_BE16(0xffff),
3487         };
3488         void *headers_m;
3489         void *headers_v;
3490         char *l24_v;
3491         unsigned int i;
3492
3493         if (!eth_v)
3494                 return;
3495         if (!eth_m)
3496                 eth_m = &nic_mask;
3497         if (inner) {
3498                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3499                                          inner_headers);
3500                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3501         } else {
3502                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3503                                          outer_headers);
3504                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3505         }
3506         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
3507                &eth_m->dst, sizeof(eth_m->dst));
3508         /* The value must be in the range of the mask. */
3509         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
3510         for (i = 0; i < sizeof(eth_m->dst); ++i)
3511                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
3512         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
3513                &eth_m->src, sizeof(eth_m->src));
3514         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
3515         /* The value must be in the range of the mask. */
3516         for (i = 0; i < sizeof(eth_m->dst); ++i)
3517                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
3518         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3519                  rte_be_to_cpu_16(eth_m->type));
3520         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
3521         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
3522 }
3523
3524 /**
3525  * Add VLAN item to matcher and to the value.
3526  *
3527  * @param[in, out] dev_flow
3528  *   Flow descriptor.
3529  * @param[in, out] matcher
3530  *   Flow matcher.
3531  * @param[in, out] key
3532  *   Flow matcher value.
3533  * @param[in] item
3534  *   Flow pattern to translate.
3535  * @param[in] inner
3536  *   Item is inner pattern.
3537  */
3538 static void
3539 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
3540                             void *matcher, void *key,
3541                             const struct rte_flow_item *item,
3542                             int inner)
3543 {
3544         const struct rte_flow_item_vlan *vlan_m = item->mask;
3545         const struct rte_flow_item_vlan *vlan_v = item->spec;
3546         void *headers_m;
3547         void *headers_v;
3548         uint16_t tci_m;
3549         uint16_t tci_v;
3550
3551         if (!vlan_v)
3552                 return;
3553         if (!vlan_m)
3554                 vlan_m = &rte_flow_item_vlan_mask;
3555         if (inner) {
3556                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3557                                          inner_headers);
3558                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3559         } else {
3560                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3561                                          outer_headers);
3562                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3563                 /*
3564                  * This is workaround, masks are not supported,
3565                  * and pre-validated.
3566                  */
3567                 dev_flow->dv.vf_vlan.tag =
3568                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
3569         }
3570         tci_m = rte_be_to_cpu_16(vlan_m->tci);
3571         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
3572         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
3573         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
3574         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
3575         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
3576         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
3577         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
3578         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
3579         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
3580         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3581                  rte_be_to_cpu_16(vlan_m->inner_type));
3582         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
3583                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
3584 }
3585
3586 /**
3587  * Add IPV4 item to matcher and to the value.
3588  *
3589  * @param[in, out] matcher
3590  *   Flow matcher.
3591  * @param[in, out] key
3592  *   Flow matcher value.
3593  * @param[in] item
3594  *   Flow pattern to translate.
3595  * @param[in] inner
3596  *   Item is inner pattern.
3597  * @param[in] group
3598  *   The group to insert the rule.
3599  */
3600 static void
3601 flow_dv_translate_item_ipv4(void *matcher, void *key,
3602                             const struct rte_flow_item *item,
3603                             int inner, uint32_t group)
3604 {
3605         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
3606         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
3607         const struct rte_flow_item_ipv4 nic_mask = {
3608                 .hdr = {
3609                         .src_addr = RTE_BE32(0xffffffff),
3610                         .dst_addr = RTE_BE32(0xffffffff),
3611                         .type_of_service = 0xff,
3612                         .next_proto_id = 0xff,
3613                 },
3614         };
3615         void *headers_m;
3616         void *headers_v;
3617         char *l24_m;
3618         char *l24_v;
3619         uint8_t tos;
3620
3621         if (inner) {
3622                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3623                                          inner_headers);
3624                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3625         } else {
3626                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3627                                          outer_headers);
3628                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3629         }
3630         if (group == 0)
3631                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3632         else
3633                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
3634         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
3635         if (!ipv4_v)
3636                 return;
3637         if (!ipv4_m)
3638                 ipv4_m = &nic_mask;
3639         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3640                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3641         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3642                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3643         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
3644         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
3645         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3646                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
3647         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3648                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
3649         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
3650         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
3651         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
3652         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
3653                  ipv4_m->hdr.type_of_service);
3654         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
3655         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
3656                  ipv4_m->hdr.type_of_service >> 2);
3657         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
3658         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3659                  ipv4_m->hdr.next_proto_id);
3660         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3661                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3662 }
3663
3664 /**
3665  * Add IPV6 item to matcher and to the value.
3666  *
3667  * @param[in, out] matcher
3668  *   Flow matcher.
3669  * @param[in, out] key
3670  *   Flow matcher value.
3671  * @param[in] item
3672  *   Flow pattern to translate.
3673  * @param[in] inner
3674  *   Item is inner pattern.
3675  * @param[in] group
3676  *   The group to insert the rule.
3677  */
3678 static void
3679 flow_dv_translate_item_ipv6(void *matcher, void *key,
3680                             const struct rte_flow_item *item,
3681                             int inner, uint32_t group)
3682 {
3683         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
3684         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
3685         const struct rte_flow_item_ipv6 nic_mask = {
3686                 .hdr = {
3687                         .src_addr =
3688                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
3689                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
3690                         .dst_addr =
3691                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
3692                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
3693                         .vtc_flow = RTE_BE32(0xffffffff),
3694                         .proto = 0xff,
3695                         .hop_limits = 0xff,
3696                 },
3697         };
3698         void *headers_m;
3699         void *headers_v;
3700         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3701         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3702         char *l24_m;
3703         char *l24_v;
3704         uint32_t vtc_m;
3705         uint32_t vtc_v;
3706         int i;
3707         int size;
3708
3709         if (inner) {
3710                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3711                                          inner_headers);
3712                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3713         } else {
3714                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3715                                          outer_headers);
3716                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3717         }
3718         if (group == 0)
3719                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3720         else
3721                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
3722         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
3723         if (!ipv6_v)
3724                 return;
3725         if (!ipv6_m)
3726                 ipv6_m = &nic_mask;
3727         size = sizeof(ipv6_m->hdr.dst_addr);
3728         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3729                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3730         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3731                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3732         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
3733         for (i = 0; i < size; ++i)
3734                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
3735         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3736                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
3737         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3738                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
3739         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
3740         for (i = 0; i < size; ++i)
3741                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
3742         /* TOS. */
3743         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
3744         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
3745         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
3746         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
3747         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
3748         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
3749         /* Label. */
3750         if (inner) {
3751                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
3752                          vtc_m);
3753                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
3754                          vtc_v);
3755         } else {
3756                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
3757                          vtc_m);
3758                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
3759                          vtc_v);
3760         }
3761         /* Protocol. */
3762         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3763                  ipv6_m->hdr.proto);
3764         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3765                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
3766 }
3767
3768 /**
3769  * Add TCP item to matcher and to the value.
3770  *
3771  * @param[in, out] matcher
3772  *   Flow matcher.
3773  * @param[in, out] key
3774  *   Flow matcher value.
3775  * @param[in] item
3776  *   Flow pattern to translate.
3777  * @param[in] inner
3778  *   Item is inner pattern.
3779  */
3780 static void
3781 flow_dv_translate_item_tcp(void *matcher, void *key,
3782                            const struct rte_flow_item *item,
3783                            int inner)
3784 {
3785         const struct rte_flow_item_tcp *tcp_m = item->mask;
3786         const struct rte_flow_item_tcp *tcp_v = item->spec;
3787         void *headers_m;
3788         void *headers_v;
3789
3790         if (inner) {
3791                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3792                                          inner_headers);
3793                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3794         } else {
3795                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3796                                          outer_headers);
3797                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3798         }
3799         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3800         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
3801         if (!tcp_v)
3802                 return;
3803         if (!tcp_m)
3804                 tcp_m = &rte_flow_item_tcp_mask;
3805         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
3806                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
3807         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
3808                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
3809         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
3810                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
3811         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
3812                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
3813         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
3814                  tcp_m->hdr.tcp_flags);
3815         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
3816                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
3817 }
3818
3819 /**
3820  * Add UDP item to matcher and to the value.
3821  *
3822  * @param[in, out] matcher
3823  *   Flow matcher.
3824  * @param[in, out] key
3825  *   Flow matcher value.
3826  * @param[in] item
3827  *   Flow pattern to translate.
3828  * @param[in] inner
3829  *   Item is inner pattern.
3830  */
3831 static void
3832 flow_dv_translate_item_udp(void *matcher, void *key,
3833                            const struct rte_flow_item *item,
3834                            int inner)
3835 {
3836         const struct rte_flow_item_udp *udp_m = item->mask;
3837         const struct rte_flow_item_udp *udp_v = item->spec;
3838         void *headers_m;
3839         void *headers_v;
3840
3841         if (inner) {
3842                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3843                                          inner_headers);
3844                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3845         } else {
3846                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3847                                          outer_headers);
3848                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3849         }
3850         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3851         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
3852         if (!udp_v)
3853                 return;
3854         if (!udp_m)
3855                 udp_m = &rte_flow_item_udp_mask;
3856         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
3857                  rte_be_to_cpu_16(udp_m->hdr.src_port));
3858         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
3859                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
3860         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
3861                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
3862         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
3863                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
3864 }
3865
3866 /**
3867  * Add GRE optional Key item to matcher and to the value.
3868  *
3869  * @param[in, out] matcher
3870  *   Flow matcher.
3871  * @param[in, out] key
3872  *   Flow matcher value.
3873  * @param[in] item
3874  *   Flow pattern to translate.
3875  * @param[in] inner
3876  *   Item is inner pattern.
3877  */
3878 static void
3879 flow_dv_translate_item_gre_key(void *matcher, void *key,
3880                                    const struct rte_flow_item *item)
3881 {
3882         const rte_be32_t *key_m = item->mask;
3883         const rte_be32_t *key_v = item->spec;
3884         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3885         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3886         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
3887
3888         if (!key_v)
3889                 return;
3890         if (!key_m)
3891                 key_m = &gre_key_default_mask;
3892         /* GRE K bit must be on and should already be validated */
3893         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
3894         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
3895         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
3896                  rte_be_to_cpu_32(*key_m) >> 8);
3897         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
3898                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
3899         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
3900                  rte_be_to_cpu_32(*key_m) & 0xFF);
3901         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
3902                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
3903 }
3904
3905 /**
3906  * Add GRE item to matcher and to the value.
3907  *
3908  * @param[in, out] matcher
3909  *   Flow matcher.
3910  * @param[in, out] key
3911  *   Flow matcher value.
3912  * @param[in] item
3913  *   Flow pattern to translate.
3914  * @param[in] inner
3915  *   Item is inner pattern.
3916  */
3917 static void
3918 flow_dv_translate_item_gre(void *matcher, void *key,
3919                            const struct rte_flow_item *item,
3920                            int inner)
3921 {
3922         const struct rte_flow_item_gre *gre_m = item->mask;
3923         const struct rte_flow_item_gre *gre_v = item->spec;
3924         void *headers_m;
3925         void *headers_v;
3926         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3927         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3928         struct {
3929                 union {
3930                         __extension__
3931                         struct {
3932                                 uint16_t version:3;
3933                                 uint16_t rsvd0:9;
3934                                 uint16_t s_present:1;
3935                                 uint16_t k_present:1;
3936                                 uint16_t rsvd_bit1:1;
3937                                 uint16_t c_present:1;
3938                         };
3939                         uint16_t value;
3940                 };
3941         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
3942
3943         if (inner) {
3944                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3945                                          inner_headers);
3946                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3947         } else {
3948                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3949                                          outer_headers);
3950                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3951         }
3952         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3953         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
3954         if (!gre_v)
3955                 return;
3956         if (!gre_m)
3957                 gre_m = &rte_flow_item_gre_mask;
3958         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
3959                  rte_be_to_cpu_16(gre_m->protocol));
3960         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
3961                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
3962         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
3963         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
3964         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
3965                  gre_crks_rsvd0_ver_m.c_present);
3966         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
3967                  gre_crks_rsvd0_ver_v.c_present &
3968                  gre_crks_rsvd0_ver_m.c_present);
3969         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
3970                  gre_crks_rsvd0_ver_m.k_present);
3971         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
3972                  gre_crks_rsvd0_ver_v.k_present &
3973                  gre_crks_rsvd0_ver_m.k_present);
3974         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
3975                  gre_crks_rsvd0_ver_m.s_present);
3976         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
3977                  gre_crks_rsvd0_ver_v.s_present &
3978                  gre_crks_rsvd0_ver_m.s_present);
3979 }
3980
3981 /**
3982  * Add NVGRE item to matcher and to the value.
3983  *
3984  * @param[in, out] matcher
3985  *   Flow matcher.
3986  * @param[in, out] key
3987  *   Flow matcher value.
3988  * @param[in] item
3989  *   Flow pattern to translate.
3990  * @param[in] inner
3991  *   Item is inner pattern.
3992  */
3993 static void
3994 flow_dv_translate_item_nvgre(void *matcher, void *key,
3995                              const struct rte_flow_item *item,
3996                              int inner)
3997 {
3998         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
3999         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
4000         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4001         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4002         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
4003         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
4004         char *gre_key_m;
4005         char *gre_key_v;
4006         int size;
4007         int i;
4008
4009         /* For NVGRE, GRE header fields must be set with defined values. */
4010         const struct rte_flow_item_gre gre_spec = {
4011                 .c_rsvd0_ver = RTE_BE16(0x2000),
4012                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
4013         };
4014         const struct rte_flow_item_gre gre_mask = {
4015                 .c_rsvd0_ver = RTE_BE16(0xB000),
4016                 .protocol = RTE_BE16(UINT16_MAX),
4017         };
4018         const struct rte_flow_item gre_item = {
4019                 .spec = &gre_spec,
4020                 .mask = &gre_mask,
4021                 .last = NULL,
4022         };
4023         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
4024         if (!nvgre_v)
4025                 return;
4026         if (!nvgre_m)
4027                 nvgre_m = &rte_flow_item_nvgre_mask;
4028         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
4029         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
4030         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
4031         memcpy(gre_key_m, tni_flow_id_m, size);
4032         for (i = 0; i < size; ++i)
4033                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
4034 }
4035
4036 /**
4037  * Add VXLAN item to matcher and to the value.
4038  *
4039  * @param[in, out] matcher
4040  *   Flow matcher.
4041  * @param[in, out] key
4042  *   Flow matcher value.
4043  * @param[in] item
4044  *   Flow pattern to translate.
4045  * @param[in] inner
4046  *   Item is inner pattern.
4047  */
4048 static void
4049 flow_dv_translate_item_vxlan(void *matcher, void *key,
4050                              const struct rte_flow_item *item,
4051                              int inner)
4052 {
4053         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
4054         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
4055         void *headers_m;
4056         void *headers_v;
4057         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4058         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4059         char *vni_m;
4060         char *vni_v;
4061         uint16_t dport;
4062         int size;
4063         int i;
4064
4065         if (inner) {
4066                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4067                                          inner_headers);
4068                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4069         } else {
4070                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4071                                          outer_headers);
4072                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4073         }
4074         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
4075                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
4076         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
4077                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
4078                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
4079         }
4080         if (!vxlan_v)
4081                 return;
4082         if (!vxlan_m)
4083                 vxlan_m = &rte_flow_item_vxlan_mask;
4084         size = sizeof(vxlan_m->vni);
4085         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4086         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4087         memcpy(vni_m, vxlan_m->vni, size);
4088         for (i = 0; i < size; ++i)
4089                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4090 }
4091
4092 /**
4093  * Add MPLS item to matcher and to the value.
4094  *
4095  * @param[in, out] matcher
4096  *   Flow matcher.
4097  * @param[in, out] key
4098  *   Flow matcher value.
4099  * @param[in] item
4100  *   Flow pattern to translate.
4101  * @param[in] prev_layer
4102  *   The protocol layer indicated in previous item.
4103  * @param[in] inner
4104  *   Item is inner pattern.
4105  */
4106 static void
4107 flow_dv_translate_item_mpls(void *matcher, void *key,
4108                             const struct rte_flow_item *item,
4109                             uint64_t prev_layer,
4110                             int inner)
4111 {
4112         const uint32_t *in_mpls_m = item->mask;
4113         const uint32_t *in_mpls_v = item->spec;
4114         uint32_t *out_mpls_m = 0;
4115         uint32_t *out_mpls_v = 0;
4116         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4117         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4118         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4119                                      misc_parameters_2);
4120         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4121         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4122         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4123
4124         switch (prev_layer) {
4125         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4126                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4127                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4128                          MLX5_UDP_PORT_MPLS);
4129                 break;
4130         case MLX5_FLOW_LAYER_GRE:
4131                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4132                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4133                          RTE_ETHER_TYPE_MPLS);
4134                 break;
4135         default:
4136                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4137                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4138                          IPPROTO_MPLS);
4139                 break;
4140         }
4141         if (!in_mpls_v)
4142                 return;
4143         if (!in_mpls_m)
4144                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4145         switch (prev_layer) {
4146         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4147                 out_mpls_m =
4148                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4149                                                  outer_first_mpls_over_udp);
4150                 out_mpls_v =
4151                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4152                                                  outer_first_mpls_over_udp);
4153                 break;
4154         case MLX5_FLOW_LAYER_GRE:
4155                 out_mpls_m =
4156                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4157                                                  outer_first_mpls_over_gre);
4158                 out_mpls_v =
4159                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4160                                                  outer_first_mpls_over_gre);
4161                 break;
4162         default:
4163                 /* Inner MPLS not over GRE is not supported. */
4164                 if (!inner) {
4165                         out_mpls_m =
4166                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4167                                                          misc2_m,
4168                                                          outer_first_mpls);
4169                         out_mpls_v =
4170                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4171                                                          misc2_v,
4172                                                          outer_first_mpls);
4173                 }
4174                 break;
4175         }
4176         if (out_mpls_m && out_mpls_v) {
4177                 *out_mpls_m = *in_mpls_m;
4178                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4179         }
4180 }
4181
4182 /**
4183  * Add META item to matcher
4184  *
4185  * @param[in, out] matcher
4186  *   Flow matcher.
4187  * @param[in, out] key
4188  *   Flow matcher value.
4189  * @param[in] item
4190  *   Flow pattern to translate.
4191  * @param[in] inner
4192  *   Item is inner pattern.
4193  */
4194 static void
4195 flow_dv_translate_item_meta(void *matcher, void *key,
4196                             const struct rte_flow_item *item)
4197 {
4198         const struct rte_flow_item_meta *meta_m;
4199         const struct rte_flow_item_meta *meta_v;
4200         void *misc2_m =
4201                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4202         void *misc2_v =
4203                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4204
4205         meta_m = (const void *)item->mask;
4206         if (!meta_m)
4207                 meta_m = &rte_flow_item_meta_mask;
4208         meta_v = (const void *)item->spec;
4209         if (meta_v) {
4210                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
4211                          rte_be_to_cpu_32(meta_m->data));
4212                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
4213                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
4214         }
4215 }
4216
4217 /**
4218  * Add source vport match to the specified matcher.
4219  *
4220  * @param[in, out] matcher
4221  *   Flow matcher.
4222  * @param[in, out] key
4223  *   Flow matcher value.
4224  * @param[in] port
4225  *   Source vport value to match
4226  * @param[in] mask
4227  *   Mask
4228  */
4229 static void
4230 flow_dv_translate_item_source_vport(void *matcher, void *key,
4231                                     int16_t port, uint16_t mask)
4232 {
4233         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4234         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4235
4236         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
4237         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
4238 }
4239
4240 /**
4241  * Translate port-id item to eswitch match on  port-id.
4242  *
4243  * @param[in] dev
4244  *   The devich to configure through.
4245  * @param[in, out] matcher
4246  *   Flow matcher.
4247  * @param[in, out] key
4248  *   Flow matcher value.
4249  * @param[in] item
4250  *   Flow pattern to translate.
4251  *
4252  * @return
4253  *   0 on success, a negative errno value otherwise.
4254  */
4255 static int
4256 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
4257                                void *key, const struct rte_flow_item *item)
4258 {
4259         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
4260         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
4261         uint16_t mask, val, id;
4262         int ret;
4263
4264         mask = pid_m ? pid_m->id : 0xffff;
4265         id = pid_v ? pid_v->id : dev->data->port_id;
4266         ret = mlx5_port_to_eswitch_info(id, NULL, &val);
4267         if (ret)
4268                 return ret;
4269         flow_dv_translate_item_source_vport(matcher, key, val, mask);
4270         return 0;
4271 }
4272
4273 /**
4274  * Add ICMP6 item to matcher and to the value.
4275  *
4276  * @param[in, out] matcher
4277  *   Flow matcher.
4278  * @param[in, out] key
4279  *   Flow matcher value.
4280  * @param[in] item
4281  *   Flow pattern to translate.
4282  * @param[in] inner
4283  *   Item is inner pattern.
4284  */
4285 static void
4286 flow_dv_translate_item_icmp6(void *matcher, void *key,
4287                               const struct rte_flow_item *item,
4288                               int inner)
4289 {
4290         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
4291         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
4292         void *headers_m;
4293         void *headers_v;
4294         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4295                                      misc_parameters_3);
4296         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4297         if (inner) {
4298                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4299                                          inner_headers);
4300                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4301         } else {
4302                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4303                                          outer_headers);
4304                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4305         }
4306         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4307         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
4308         if (!icmp6_v)
4309                 return;
4310         if (!icmp6_m)
4311                 icmp6_m = &rte_flow_item_icmp6_mask;
4312         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
4313         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
4314                  icmp6_v->type & icmp6_m->type);
4315         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
4316         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
4317                  icmp6_v->code & icmp6_m->code);
4318 }
4319
4320 /**
4321  * Add ICMP item to matcher and to the value.
4322  *
4323  * @param[in, out] matcher
4324  *   Flow matcher.
4325  * @param[in, out] key
4326  *   Flow matcher value.
4327  * @param[in] item
4328  *   Flow pattern to translate.
4329  * @param[in] inner
4330  *   Item is inner pattern.
4331  */
4332 static void
4333 flow_dv_translate_item_icmp(void *matcher, void *key,
4334                             const struct rte_flow_item *item,
4335                             int inner)
4336 {
4337         const struct rte_flow_item_icmp *icmp_m = item->mask;
4338         const struct rte_flow_item_icmp *icmp_v = item->spec;
4339         void *headers_m;
4340         void *headers_v;
4341         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4342                                      misc_parameters_3);
4343         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4344         if (inner) {
4345                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4346                                          inner_headers);
4347                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4348         } else {
4349                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4350                                          outer_headers);
4351                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4352         }
4353         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4354         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
4355         if (!icmp_v)
4356                 return;
4357         if (!icmp_m)
4358                 icmp_m = &rte_flow_item_icmp_mask;
4359         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
4360                  icmp_m->hdr.icmp_type);
4361         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
4362                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
4363         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
4364                  icmp_m->hdr.icmp_code);
4365         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
4366                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
4367 }
4368
4369 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
4370
4371 #define HEADER_IS_ZERO(match_criteria, headers)                              \
4372         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
4373                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
4374
4375 /**
4376  * Calculate flow matcher enable bitmap.
4377  *
4378  * @param match_criteria
4379  *   Pointer to flow matcher criteria.
4380  *
4381  * @return
4382  *   Bitmap of enabled fields.
4383  */
4384 static uint8_t
4385 flow_dv_matcher_enable(uint32_t *match_criteria)
4386 {
4387         uint8_t match_criteria_enable;
4388
4389         match_criteria_enable =
4390                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
4391                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
4392         match_criteria_enable |=
4393                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
4394                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
4395         match_criteria_enable |=
4396                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
4397                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
4398         match_criteria_enable |=
4399                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
4400                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
4401         match_criteria_enable |=
4402                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
4403                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
4404         return match_criteria_enable;
4405 }
4406
4407
4408 /**
4409  * Get a flow table.
4410  *
4411  * @param dev[in, out]
4412  *   Pointer to rte_eth_dev structure.
4413  * @param[in] table_id
4414  *   Table id to use.
4415  * @param[in] egress
4416  *   Direction of the table.
4417  * @param[in] transfer
4418  *   E-Switch or NIC flow.
4419  * @param[out] error
4420  *   pointer to error structure.
4421  *
4422  * @return
4423  *   Returns tables resource based on the index, NULL in case of failed.
4424  */
4425 static struct mlx5_flow_tbl_resource *
4426 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
4427                          uint32_t table_id, uint8_t egress,
4428                          uint8_t transfer,
4429                          struct rte_flow_error *error)
4430 {
4431         struct mlx5_priv *priv = dev->data->dev_private;
4432         struct mlx5_ibv_shared *sh = priv->sh;
4433         struct mlx5_flow_tbl_resource *tbl;
4434
4435 #ifdef HAVE_MLX5DV_DR
4436         if (transfer) {
4437                 tbl = &sh->fdb_tbl[table_id];
4438                 if (!tbl->obj)
4439                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4440                                 (sh->fdb_domain, table_id);
4441         } else if (egress) {
4442                 tbl = &sh->tx_tbl[table_id];
4443                 if (!tbl->obj)
4444                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4445                                 (sh->tx_domain, table_id);
4446         } else {
4447                 tbl = &sh->rx_tbl[table_id];
4448                 if (!tbl->obj)
4449                         tbl->obj = mlx5_glue->dr_create_flow_tbl
4450                                 (sh->rx_domain, table_id);
4451         }
4452         if (!tbl->obj) {
4453                 rte_flow_error_set(error, ENOMEM,
4454                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4455                                    NULL, "cannot create table");
4456                 return NULL;
4457         }
4458         rte_atomic32_inc(&tbl->refcnt);
4459         return tbl;
4460 #else
4461         (void)error;
4462         (void)tbl;
4463         if (transfer)
4464                 return &sh->fdb_tbl[table_id];
4465         else if (egress)
4466                 return &sh->tx_tbl[table_id];
4467         else
4468                 return &sh->rx_tbl[table_id];
4469 #endif
4470 }
4471
4472 /**
4473  * Release a flow table.
4474  *
4475  * @param[in] tbl
4476  *   Table resource to be released.
4477  *
4478  * @return
4479  *   Returns 0 if table was released, else return 1;
4480  */
4481 static int
4482 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
4483 {
4484         if (!tbl)
4485                 return 0;
4486         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
4487                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
4488                 tbl->obj = NULL;
4489                 return 0;
4490         }
4491         return 1;
4492 }
4493
4494 /**
4495  * Register the flow matcher.
4496  *
4497  * @param dev[in, out]
4498  *   Pointer to rte_eth_dev structure.
4499  * @param[in, out] matcher
4500  *   Pointer to flow matcher.
4501  * @parm[in, out] dev_flow
4502  *   Pointer to the dev_flow.
4503  * @param[out] error
4504  *   pointer to error structure.
4505  *
4506  * @return
4507  *   0 on success otherwise -errno and errno is set.
4508  */
4509 static int
4510 flow_dv_matcher_register(struct rte_eth_dev *dev,
4511                          struct mlx5_flow_dv_matcher *matcher,
4512                          struct mlx5_flow *dev_flow,
4513                          struct rte_flow_error *error)
4514 {
4515         struct mlx5_priv *priv = dev->data->dev_private;
4516         struct mlx5_ibv_shared *sh = priv->sh;
4517         struct mlx5_flow_dv_matcher *cache_matcher;
4518         struct mlx5dv_flow_matcher_attr dv_attr = {
4519                 .type = IBV_FLOW_ATTR_NORMAL,
4520                 .match_mask = (void *)&matcher->mask,
4521         };
4522         struct mlx5_flow_tbl_resource *tbl = NULL;
4523
4524         /* Lookup from cache. */
4525         LIST_FOREACH(cache_matcher, &sh->matchers, next) {
4526                 if (matcher->crc == cache_matcher->crc &&
4527                     matcher->priority == cache_matcher->priority &&
4528                     matcher->egress == cache_matcher->egress &&
4529                     matcher->group == cache_matcher->group &&
4530                     matcher->transfer == cache_matcher->transfer &&
4531                     !memcmp((const void *)matcher->mask.buf,
4532                             (const void *)cache_matcher->mask.buf,
4533                             cache_matcher->mask.size)) {
4534                         DRV_LOG(DEBUG,
4535                                 "priority %hd use %s matcher %p: refcnt %d++",
4536                                 cache_matcher->priority,
4537                                 cache_matcher->egress ? "tx" : "rx",
4538                                 (void *)cache_matcher,
4539                                 rte_atomic32_read(&cache_matcher->refcnt));
4540                         rte_atomic32_inc(&cache_matcher->refcnt);
4541                         dev_flow->dv.matcher = cache_matcher;
4542                         return 0;
4543                 }
4544         }
4545         /* Register new matcher. */
4546         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
4547         if (!cache_matcher)
4548                 return rte_flow_error_set(error, ENOMEM,
4549                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4550                                           "cannot allocate matcher memory");
4551         tbl = flow_dv_tbl_resource_get(dev, matcher->group,
4552                                        matcher->egress, matcher->transfer,
4553                                        error);
4554         if (!tbl) {
4555                 rte_free(cache_matcher);
4556                 return rte_flow_error_set(error, ENOMEM,
4557                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4558                                           NULL, "cannot create table");
4559         }
4560         *cache_matcher = *matcher;
4561         dv_attr.match_criteria_enable =
4562                 flow_dv_matcher_enable(cache_matcher->mask.buf);
4563         dv_attr.priority = matcher->priority;
4564         if (matcher->egress)
4565                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
4566         cache_matcher->matcher_object =
4567                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
4568         if (!cache_matcher->matcher_object) {
4569                 rte_free(cache_matcher);
4570 #ifdef HAVE_MLX5DV_DR
4571                 flow_dv_tbl_resource_release(tbl);
4572 #endif
4573                 return rte_flow_error_set(error, ENOMEM,
4574                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4575                                           NULL, "cannot create matcher");
4576         }
4577         rte_atomic32_inc(&cache_matcher->refcnt);
4578         LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
4579         dev_flow->dv.matcher = cache_matcher;
4580         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
4581                 cache_matcher->priority,
4582                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
4583                 rte_atomic32_read(&cache_matcher->refcnt));
4584         rte_atomic32_inc(&tbl->refcnt);
4585         return 0;
4586 }
4587
4588 /**
4589  * Find existing tag resource or create and register a new one.
4590  *
4591  * @param dev[in, out]
4592  *   Pointer to rte_eth_dev structure.
4593  * @param[in, out] resource
4594  *   Pointer to tag resource.
4595  * @parm[in, out] dev_flow
4596  *   Pointer to the dev_flow.
4597  * @param[out] error
4598  *   pointer to error structure.
4599  *
4600  * @return
4601  *   0 on success otherwise -errno and errno is set.
4602  */
4603 static int
4604 flow_dv_tag_resource_register
4605                         (struct rte_eth_dev *dev,
4606                          struct mlx5_flow_dv_tag_resource *resource,
4607                          struct mlx5_flow *dev_flow,
4608                          struct rte_flow_error *error)
4609 {
4610         struct mlx5_priv *priv = dev->data->dev_private;
4611         struct mlx5_ibv_shared *sh = priv->sh;
4612         struct mlx5_flow_dv_tag_resource *cache_resource;
4613
4614         /* Lookup a matching resource from cache. */
4615         LIST_FOREACH(cache_resource, &sh->tags, next) {
4616                 if (resource->tag == cache_resource->tag) {
4617                         DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
4618                                 (void *)cache_resource,
4619                                 rte_atomic32_read(&cache_resource->refcnt));
4620                         rte_atomic32_inc(&cache_resource->refcnt);
4621                         dev_flow->flow->tag_resource = cache_resource;
4622                         return 0;
4623                 }
4624         }
4625         /* Register new  resource. */
4626         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
4627         if (!cache_resource)
4628                 return rte_flow_error_set(error, ENOMEM,
4629                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4630                                           "cannot allocate resource memory");
4631         *cache_resource = *resource;
4632         cache_resource->action = mlx5_glue->dv_create_flow_action_tag
4633                 (resource->tag);
4634         if (!cache_resource->action) {
4635                 rte_free(cache_resource);
4636                 return rte_flow_error_set(error, ENOMEM,
4637                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4638                                           NULL, "cannot create action");
4639         }
4640         rte_atomic32_init(&cache_resource->refcnt);
4641         rte_atomic32_inc(&cache_resource->refcnt);
4642         LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
4643         dev_flow->flow->tag_resource = cache_resource;
4644         DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
4645                 (void *)cache_resource,
4646                 rte_atomic32_read(&cache_resource->refcnt));
4647         return 0;
4648 }
4649
4650 /**
4651  * Release the tag.
4652  *
4653  * @param dev
4654  *   Pointer to Ethernet device.
4655  * @param flow
4656  *   Pointer to mlx5_flow.
4657  *
4658  * @return
4659  *   1 while a reference on it exists, 0 when freed.
4660  */
4661 static int
4662 flow_dv_tag_release(struct rte_eth_dev *dev,
4663                     struct mlx5_flow_dv_tag_resource *tag)
4664 {
4665         assert(tag);
4666         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
4667                 dev->data->port_id, (void *)tag,
4668                 rte_atomic32_read(&tag->refcnt));
4669         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
4670                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
4671                 LIST_REMOVE(tag, next);
4672                 DRV_LOG(DEBUG, "port %u tag %p: removed",
4673                         dev->data->port_id, (void *)tag);
4674                 rte_free(tag);
4675                 return 0;
4676         }
4677         return 1;
4678 }
4679
4680 /**
4681  * Translate port ID action to vport.
4682  *
4683  * @param[in] dev
4684  *   Pointer to rte_eth_dev structure.
4685  * @param[in] action
4686  *   Pointer to the port ID action.
4687  * @param[out] dst_port_id
4688  *   The target port ID.
4689  * @param[out] error
4690  *   Pointer to the error structure.
4691  *
4692  * @return
4693  *   0 on success, a negative errno value otherwise and rte_errno is set.
4694  */
4695 static int
4696 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
4697                                  const struct rte_flow_action *action,
4698                                  uint32_t *dst_port_id,
4699                                  struct rte_flow_error *error)
4700 {
4701         uint32_t port;
4702         uint16_t port_id;
4703         int ret;
4704         const struct rte_flow_action_port_id *conf =
4705                         (const struct rte_flow_action_port_id *)action->conf;
4706
4707         port = conf->original ? dev->data->port_id : conf->id;
4708         ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
4709         if (ret)
4710                 return rte_flow_error_set(error, -ret,
4711                                           RTE_FLOW_ERROR_TYPE_ACTION,
4712                                           NULL,
4713                                           "No eswitch info was found for port");
4714         *dst_port_id = port_id;
4715         return 0;
4716 }
4717
4718 /**
4719  * Fill the flow with DV spec.
4720  *
4721  * @param[in] dev
4722  *   Pointer to rte_eth_dev structure.
4723  * @param[in, out] dev_flow
4724  *   Pointer to the sub flow.
4725  * @param[in] attr
4726  *   Pointer to the flow attributes.
4727  * @param[in] items
4728  *   Pointer to the list of items.
4729  * @param[in] actions
4730  *   Pointer to the list of actions.
4731  * @param[out] error
4732  *   Pointer to the error structure.
4733  *
4734  * @return
4735  *   0 on success, a negative errno value otherwise and rte_errno is set.
4736  */
4737 static int
4738 flow_dv_translate(struct rte_eth_dev *dev,
4739                   struct mlx5_flow *dev_flow,
4740                   const struct rte_flow_attr *attr,
4741                   const struct rte_flow_item items[],
4742                   const struct rte_flow_action actions[],
4743                   struct rte_flow_error *error)
4744 {
4745         struct mlx5_priv *priv = dev->data->dev_private;
4746         struct rte_flow *flow = dev_flow->flow;
4747         uint64_t item_flags = 0;
4748         uint64_t last_item = 0;
4749         uint64_t action_flags = 0;
4750         uint64_t priority = attr->priority;
4751         struct mlx5_flow_dv_matcher matcher = {
4752                 .mask = {
4753                         .size = sizeof(matcher.mask.buf),
4754                 },
4755         };
4756         int actions_n = 0;
4757         bool actions_end = false;
4758         struct mlx5_flow_dv_modify_hdr_resource res = {
4759                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4760                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
4761         };
4762         union flow_dv_attr flow_attr = { .attr = 0 };
4763         struct mlx5_flow_dv_tag_resource tag_resource;
4764         uint32_t modify_action_position = UINT32_MAX;
4765         void *match_mask = matcher.mask.buf;
4766         void *match_value = dev_flow->dv.value.buf;
4767         uint8_t next_protocol = 0xff;
4768
4769         flow->group = attr->group;
4770         if (attr->transfer)
4771                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4772         if (priority == MLX5_FLOW_PRIO_RSVD)
4773                 priority = priv->config.flow_prio - 1;
4774         for (; !actions_end ; actions++) {
4775                 const struct rte_flow_action_queue *queue;
4776                 const struct rte_flow_action_rss *rss;
4777                 const struct rte_flow_action *action = actions;
4778                 const struct rte_flow_action_count *count = action->conf;
4779                 const uint8_t *rss_key;
4780                 const struct rte_flow_action_jump *jump_data;
4781                 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
4782                 struct mlx5_flow_tbl_resource *tbl;
4783                 uint32_t port_id = 0;
4784                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
4785
4786                 switch (actions->type) {
4787                 case RTE_FLOW_ACTION_TYPE_VOID:
4788                         break;
4789                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4790                         if (flow_dv_translate_action_port_id(dev, action,
4791                                                              &port_id, error))
4792                                 return -rte_errno;
4793                         port_id_resource.port_id = port_id;
4794                         if (flow_dv_port_id_action_resource_register
4795                             (dev, &port_id_resource, dev_flow, error))
4796                                 return -rte_errno;
4797                         dev_flow->dv.actions[actions_n++] =
4798                                 dev_flow->dv.port_id_action->action;
4799                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4800                         break;
4801                 case RTE_FLOW_ACTION_TYPE_FLAG:
4802                         tag_resource.tag =
4803                                 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
4804                         if (!flow->tag_resource)
4805                                 if (flow_dv_tag_resource_register
4806                                     (dev, &tag_resource, dev_flow, error))
4807                                         return errno;
4808                         dev_flow->dv.actions[actions_n++] =
4809                                 flow->tag_resource->action;
4810                         action_flags |= MLX5_FLOW_ACTION_FLAG;
4811                         break;
4812                 case RTE_FLOW_ACTION_TYPE_MARK:
4813                         tag_resource.tag = mlx5_flow_mark_set
4814                               (((const struct rte_flow_action_mark *)
4815                                (actions->conf))->id);
4816                         if (!flow->tag_resource)
4817                                 if (flow_dv_tag_resource_register
4818                                     (dev, &tag_resource, dev_flow, error))
4819                                         return errno;
4820                         dev_flow->dv.actions[actions_n++] =
4821                                 flow->tag_resource->action;
4822                         action_flags |= MLX5_FLOW_ACTION_MARK;
4823                         break;
4824                 case RTE_FLOW_ACTION_TYPE_DROP:
4825                         action_flags |= MLX5_FLOW_ACTION_DROP;
4826                         break;
4827                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4828                         queue = actions->conf;
4829                         flow->rss.queue_num = 1;
4830                         (*flow->queue)[0] = queue->index;
4831                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
4832                         break;
4833                 case RTE_FLOW_ACTION_TYPE_RSS:
4834                         rss = actions->conf;
4835                         if (flow->queue)
4836                                 memcpy((*flow->queue), rss->queue,
4837                                        rss->queue_num * sizeof(uint16_t));
4838                         flow->rss.queue_num = rss->queue_num;
4839                         /* NULL RSS key indicates default RSS key. */
4840                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
4841                         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
4842                         /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
4843                         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
4844                         flow->rss.level = rss->level;
4845                         action_flags |= MLX5_FLOW_ACTION_RSS;
4846                         break;
4847                 case RTE_FLOW_ACTION_TYPE_COUNT:
4848                         if (!priv->config.devx) {
4849                                 rte_errno = ENOTSUP;
4850                                 goto cnt_err;
4851                         }
4852                         flow->counter = flow_dv_counter_alloc(dev,
4853                                                               count->shared,
4854                                                               count->id,
4855                                                               attr->group);
4856                         if (flow->counter == NULL)
4857                                 goto cnt_err;
4858                         dev_flow->dv.actions[actions_n++] =
4859                                 flow->counter->action;
4860                         action_flags |= MLX5_FLOW_ACTION_COUNT;
4861                         break;
4862 cnt_err:
4863                         if (rte_errno == ENOTSUP)
4864                                 return rte_flow_error_set
4865                                               (error, ENOTSUP,
4866                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4867                                                NULL,
4868                                                "count action not supported");
4869                         else
4870                                 return rte_flow_error_set
4871                                                 (error, rte_errno,
4872                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4873                                                  action,
4874                                                  "cannot create counter"
4875                                                   " object.");
4876                         break;
4877                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
4878                         dev_flow->dv.actions[actions_n++] =
4879                                                 priv->sh->pop_vlan_action;
4880                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
4881                         break;
4882                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4883                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4884                         if (flow_dv_create_action_l2_encap(dev, actions,
4885                                                            dev_flow,
4886                                                            attr->transfer,
4887                                                            error))
4888                                 return -rte_errno;
4889                         dev_flow->dv.actions[actions_n++] =
4890                                 dev_flow->dv.encap_decap->verbs_action;
4891                         action_flags |= actions->type ==
4892                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
4893                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
4894                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
4895                         break;
4896                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4897                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4898                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
4899                                                            attr->transfer,
4900                                                            error))
4901                                 return -rte_errno;
4902                         dev_flow->dv.actions[actions_n++] =
4903                                 dev_flow->dv.encap_decap->verbs_action;
4904                         action_flags |= actions->type ==
4905                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
4906                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
4907                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
4908                         break;
4909                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4910                         /* Handle encap with preceding decap. */
4911                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
4912                                 if (flow_dv_create_action_raw_encap
4913                                         (dev, actions, dev_flow, attr, error))
4914                                         return -rte_errno;
4915                                 dev_flow->dv.actions[actions_n++] =
4916                                         dev_flow->dv.encap_decap->verbs_action;
4917                         } else {
4918                                 /* Handle encap without preceding decap. */
4919                                 if (flow_dv_create_action_l2_encap
4920                                     (dev, actions, dev_flow, attr->transfer,
4921                                      error))
4922                                         return -rte_errno;
4923                                 dev_flow->dv.actions[actions_n++] =
4924                                         dev_flow->dv.encap_decap->verbs_action;
4925                         }
4926                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
4927                         break;
4928                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4929                         /* Check if this decap is followed by encap. */
4930                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
4931                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
4932                                action++) {
4933                         }
4934                         /* Handle decap only if it isn't followed by encap. */
4935                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4936                                 if (flow_dv_create_action_l2_decap
4937                                     (dev, dev_flow, attr->transfer, error))
4938                                         return -rte_errno;
4939                                 dev_flow->dv.actions[actions_n++] =
4940                                         dev_flow->dv.encap_decap->verbs_action;
4941                         }
4942                         /* If decap is followed by encap, handle it at encap. */
4943                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
4944                         break;
4945                 case RTE_FLOW_ACTION_TYPE_JUMP:
4946                         jump_data = action->conf;
4947                         tbl = flow_dv_tbl_resource_get(dev, jump_data->group,
4948                                                        attr->egress,
4949                                                        attr->transfer, error);
4950                         if (!tbl)
4951                                 return rte_flow_error_set
4952                                                 (error, errno,
4953                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4954                                                  NULL,
4955                                                  "cannot create jump action.");
4956                         jump_tbl_resource.tbl = tbl;
4957                         if (flow_dv_jump_tbl_resource_register
4958                             (dev, &jump_tbl_resource, dev_flow, error)) {
4959                                 flow_dv_tbl_resource_release(tbl);
4960                                 return rte_flow_error_set
4961                                                 (error, errno,
4962                                                  RTE_FLOW_ERROR_TYPE_ACTION,
4963                                                  NULL,
4964                                                  "cannot create jump action.");
4965                         }
4966                         dev_flow->dv.actions[actions_n++] =
4967                                 dev_flow->dv.jump->action;
4968                         action_flags |= MLX5_FLOW_ACTION_JUMP;
4969                         break;
4970                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4971                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4972                         if (flow_dv_convert_action_modify_mac(&res, actions,
4973                                                               error))
4974                                 return -rte_errno;
4975                         action_flags |= actions->type ==
4976                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
4977                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
4978                                         MLX5_FLOW_ACTION_SET_MAC_DST;
4979                         break;
4980                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4981                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4982                         if (flow_dv_convert_action_modify_ipv4(&res, actions,
4983                                                                error))
4984                                 return -rte_errno;
4985                         action_flags |= actions->type ==
4986                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
4987                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
4988                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
4989                         break;
4990                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4991                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4992                         if (flow_dv_convert_action_modify_ipv6(&res, actions,
4993                                                                error))
4994                                 return -rte_errno;
4995                         action_flags |= actions->type ==
4996                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
4997                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
4998                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
4999                         break;
5000                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5001                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5002                         if (flow_dv_convert_action_modify_tp(&res, actions,
5003                                                              items, &flow_attr,
5004                                                              error))
5005                                 return -rte_errno;
5006                         action_flags |= actions->type ==
5007                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5008                                         MLX5_FLOW_ACTION_SET_TP_SRC :
5009                                         MLX5_FLOW_ACTION_SET_TP_DST;
5010                         break;
5011                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5012                         if (flow_dv_convert_action_modify_dec_ttl(&res, items,
5013                                                                   &flow_attr,
5014                                                                   error))
5015                                 return -rte_errno;
5016                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
5017                         break;
5018                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5019                         if (flow_dv_convert_action_modify_ttl(&res, actions,
5020                                                              items, &flow_attr,
5021                                                              error))
5022                                 return -rte_errno;
5023                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
5024                         break;
5025                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5026                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5027                         if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
5028                                                                   error))
5029                                 return -rte_errno;
5030                         action_flags |= actions->type ==
5031                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5032                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
5033                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5034                         break;
5035
5036                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5037                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5038                         if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
5039                                                                   error))
5040                                 return -rte_errno;
5041                         action_flags |= actions->type ==
5042                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5043                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
5044                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
5045                         break;
5046                 case RTE_FLOW_ACTION_TYPE_END:
5047                         actions_end = true;
5048                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
5049                                 /* create modify action if needed. */
5050                                 if (flow_dv_modify_hdr_resource_register
5051                                                                 (dev, &res,
5052                                                                  dev_flow,
5053                                                                  error))
5054                                         return -rte_errno;
5055                                 dev_flow->dv.actions[modify_action_position] =
5056                                         dev_flow->dv.modify_hdr->verbs_action;
5057                         }
5058                         break;
5059                 default:
5060                         break;
5061                 }
5062                 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
5063                     modify_action_position == UINT32_MAX)
5064                         modify_action_position = actions_n++;
5065         }
5066         dev_flow->dv.actions_n = actions_n;
5067         flow->actions = action_flags;
5068         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5069                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5070
5071                 switch (items->type) {
5072                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5073                         flow_dv_translate_item_port_id(dev, match_mask,
5074                                                        match_value, items);
5075                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5076                         break;
5077                 case RTE_FLOW_ITEM_TYPE_ETH:
5078                         flow_dv_translate_item_eth(match_mask, match_value,
5079                                                    items, tunnel);
5080                         matcher.priority = MLX5_PRIORITY_MAP_L2;
5081                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5082                                              MLX5_FLOW_LAYER_OUTER_L2;
5083                         break;
5084                 case RTE_FLOW_ITEM_TYPE_VLAN:
5085                         flow_dv_translate_item_vlan(dev_flow,
5086                                                     match_mask, match_value,
5087                                                     items, tunnel);
5088                         matcher.priority = MLX5_PRIORITY_MAP_L2;
5089                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
5090                                               MLX5_FLOW_LAYER_INNER_VLAN) :
5091                                              (MLX5_FLOW_LAYER_OUTER_L2 |
5092                                               MLX5_FLOW_LAYER_OUTER_VLAN);
5093                         break;
5094                 case RTE_FLOW_ITEM_TYPE_IPV4:
5095                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5096                                                   &item_flags, &tunnel);
5097                         flow_dv_translate_item_ipv4(match_mask, match_value,
5098                                                     items, tunnel, attr->group);
5099                         matcher.priority = MLX5_PRIORITY_MAP_L3;
5100                         dev_flow->dv.hash_fields |=
5101                                 mlx5_flow_hashfields_adjust
5102                                         (dev_flow, tunnel,
5103                                          MLX5_IPV4_LAYER_TYPES,
5104                                          MLX5_IPV4_IBV_RX_HASH);
5105                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5106                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5107                         if (items->mask != NULL &&
5108                             ((const struct rte_flow_item_ipv4 *)
5109                              items->mask)->hdr.next_proto_id) {
5110                                 next_protocol =
5111                                         ((const struct rte_flow_item_ipv4 *)
5112                                          (items->spec))->hdr.next_proto_id;
5113                                 next_protocol &=
5114                                         ((const struct rte_flow_item_ipv4 *)
5115                                          (items->mask))->hdr.next_proto_id;
5116                         } else {
5117                                 /* Reset for inner layer. */
5118                                 next_protocol = 0xff;
5119                         }
5120                         break;
5121                 case RTE_FLOW_ITEM_TYPE_IPV6:
5122                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5123                                                   &item_flags, &tunnel);
5124                         flow_dv_translate_item_ipv6(match_mask, match_value,
5125                                                     items, tunnel, attr->group);
5126                         matcher.priority = MLX5_PRIORITY_MAP_L3;
5127                         dev_flow->dv.hash_fields |=
5128                                 mlx5_flow_hashfields_adjust
5129                                         (dev_flow, tunnel,
5130                                          MLX5_IPV6_LAYER_TYPES,
5131                                          MLX5_IPV6_IBV_RX_HASH);
5132                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5133                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5134                         if (items->mask != NULL &&
5135                             ((const struct rte_flow_item_ipv6 *)
5136                              items->mask)->hdr.proto) {
5137                                 next_protocol =
5138                                         ((const struct rte_flow_item_ipv6 *)
5139                                          items->spec)->hdr.proto;
5140                                 next_protocol &=
5141                                         ((const struct rte_flow_item_ipv6 *)
5142                                          items->mask)->hdr.proto;
5143                         } else {
5144                                 /* Reset for inner layer. */
5145                                 next_protocol = 0xff;
5146                         }
5147                         break;
5148                 case RTE_FLOW_ITEM_TYPE_TCP:
5149                         flow_dv_translate_item_tcp(match_mask, match_value,
5150                                                    items, tunnel);
5151                         matcher.priority = MLX5_PRIORITY_MAP_L4;
5152                         dev_flow->dv.hash_fields |=
5153                                 mlx5_flow_hashfields_adjust
5154                                         (dev_flow, tunnel, ETH_RSS_TCP,
5155                                          IBV_RX_HASH_SRC_PORT_TCP |
5156                                          IBV_RX_HASH_DST_PORT_TCP);
5157                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5158                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5159                         break;
5160                 case RTE_FLOW_ITEM_TYPE_UDP:
5161                         flow_dv_translate_item_udp(match_mask, match_value,
5162                                                    items, tunnel);
5163                         matcher.priority = MLX5_PRIORITY_MAP_L4;
5164                         dev_flow->dv.hash_fields |=
5165                                 mlx5_flow_hashfields_adjust
5166                                         (dev_flow, tunnel, ETH_RSS_UDP,
5167                                          IBV_RX_HASH_SRC_PORT_UDP |
5168                                          IBV_RX_HASH_DST_PORT_UDP);
5169                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5170                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5171                         break;
5172                 case RTE_FLOW_ITEM_TYPE_GRE:
5173                         flow_dv_translate_item_gre(match_mask, match_value,
5174                                                    items, tunnel);
5175                         last_item = MLX5_FLOW_LAYER_GRE;
5176                         break;
5177                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5178                         flow_dv_translate_item_gre_key(match_mask,
5179                                                        match_value, items);
5180                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5181                         break;
5182                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5183                         flow_dv_translate_item_nvgre(match_mask, match_value,
5184                                                      items, tunnel);
5185                         last_item = MLX5_FLOW_LAYER_GRE;
5186                         break;
5187                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5188                         flow_dv_translate_item_vxlan(match_mask, match_value,
5189                                                      items, tunnel);
5190                         last_item = MLX5_FLOW_LAYER_VXLAN;
5191                         break;
5192                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5193                         flow_dv_translate_item_vxlan(match_mask, match_value,
5194                                                      items, tunnel);
5195                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5196                         break;
5197                 case RTE_FLOW_ITEM_TYPE_MPLS:
5198                         flow_dv_translate_item_mpls(match_mask, match_value,
5199                                                     items, last_item, tunnel);
5200                         last_item = MLX5_FLOW_LAYER_MPLS;
5201                         break;
5202                 case RTE_FLOW_ITEM_TYPE_META:
5203                         flow_dv_translate_item_meta(match_mask, match_value,
5204                                                     items);
5205                         last_item = MLX5_FLOW_ITEM_METADATA;
5206                         break;
5207                 case RTE_FLOW_ITEM_TYPE_ICMP:
5208                         flow_dv_translate_item_icmp(match_mask, match_value,
5209                                                     items, tunnel);
5210                         last_item = MLX5_FLOW_LAYER_ICMP;
5211                         break;
5212                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5213                         flow_dv_translate_item_icmp6(match_mask, match_value,
5214                                                       items, tunnel);
5215                         last_item = MLX5_FLOW_LAYER_ICMP6;
5216                         break;
5217                 default:
5218                         break;
5219                 }
5220                 item_flags |= last_item;
5221         }
5222         /*
5223          * In case of ingress traffic when E-Switch mode is enabled,
5224          * we have two cases where we need to set the source port manually.
5225          * The first one, is in case of Nic steering rule, and the second is
5226          * E-Switch rule where no port_id item was found. In both cases
5227          * the source port is set according the current port in use.
5228          */
5229         if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
5230             (priv->representor || priv->master)) {
5231                 if (flow_dv_translate_item_port_id(dev, match_mask,
5232                                                    match_value, NULL))
5233                         return -rte_errno;
5234         }
5235         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
5236                                          dev_flow->dv.value.buf));
5237         dev_flow->layers = item_flags;
5238         /* Register matcher. */
5239         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
5240                                     matcher.mask.size);
5241         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
5242                                                      matcher.priority);
5243         matcher.egress = attr->egress;
5244         matcher.group = attr->group;
5245         matcher.transfer = attr->transfer;
5246         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
5247                 return -rte_errno;
5248         return 0;
5249 }
5250
5251 /**
5252  * Apply the flow to the NIC.
5253  *
5254  * @param[in] dev
5255  *   Pointer to the Ethernet device structure.
5256  * @param[in, out] flow
5257  *   Pointer to flow structure.
5258  * @param[out] error
5259  *   Pointer to error structure.
5260  *
5261  * @return
5262  *   0 on success, a negative errno value otherwise and rte_errno is set.
5263  */
5264 static int
5265 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5266               struct rte_flow_error *error)
5267 {
5268         struct mlx5_flow_dv *dv;
5269         struct mlx5_flow *dev_flow;
5270         struct mlx5_priv *priv = dev->data->dev_private;
5271         int n;
5272         int err;
5273
5274         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5275                 dv = &dev_flow->dv;
5276                 n = dv->actions_n;
5277                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
5278                         if (flow->transfer) {
5279                                 dv->actions[n++] = priv->sh->esw_drop_action;
5280                         } else {
5281                                 dv->hrxq = mlx5_hrxq_drop_new(dev);
5282                                 if (!dv->hrxq) {
5283                                         rte_flow_error_set
5284                                                 (error, errno,
5285                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5286                                                  NULL,
5287                                                  "cannot get drop hash queue");
5288                                         goto error;
5289                                 }
5290                                 dv->actions[n++] = dv->hrxq->action;
5291                         }
5292                 } else if (flow->actions &
5293                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
5294                         struct mlx5_hrxq *hrxq;
5295
5296                         hrxq = mlx5_hrxq_get(dev, flow->key,
5297                                              MLX5_RSS_HASH_KEY_LEN,
5298                                              dv->hash_fields,
5299                                              (*flow->queue),
5300                                              flow->rss.queue_num);
5301                         if (!hrxq) {
5302                                 hrxq = mlx5_hrxq_new
5303                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
5304                                          dv->hash_fields, (*flow->queue),
5305                                          flow->rss.queue_num,
5306                                          !!(dev_flow->layers &
5307                                             MLX5_FLOW_LAYER_TUNNEL));
5308                         }
5309                         if (!hrxq) {
5310                                 rte_flow_error_set
5311                                         (error, rte_errno,
5312                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5313                                          "cannot get hash queue");
5314                                 goto error;
5315                         }
5316                         dv->hrxq = hrxq;
5317                         dv->actions[n++] = dv->hrxq->action;
5318                 }
5319                 dv->flow =
5320                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
5321                                                   (void *)&dv->value, n,
5322                                                   dv->actions);
5323                 if (!dv->flow) {
5324                         rte_flow_error_set(error, errno,
5325                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5326                                            NULL,
5327                                            "hardware refuses to create flow");
5328                         goto error;
5329                 }
5330                 if (priv->vmwa_context &&
5331                     dev_flow->dv.vf_vlan.tag &&
5332                     !dev_flow->dv.vf_vlan.created) {
5333                         /*
5334                          * The rule contains the VLAN pattern.
5335                          * For VF we are going to create VLAN
5336                          * interface to make hypervisor set correct
5337                          * e-Switch vport context.
5338                          */
5339                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
5340                 }
5341         }
5342         return 0;
5343 error:
5344         err = rte_errno; /* Save rte_errno before cleanup. */
5345         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5346                 struct mlx5_flow_dv *dv = &dev_flow->dv;
5347                 if (dv->hrxq) {
5348                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
5349                                 mlx5_hrxq_drop_release(dev);
5350                         else
5351                                 mlx5_hrxq_release(dev, dv->hrxq);
5352                         dv->hrxq = NULL;
5353                 }
5354                 if (dev_flow->dv.vf_vlan.tag &&
5355                     dev_flow->dv.vf_vlan.created)
5356                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5357         }
5358         rte_errno = err; /* Restore rte_errno. */
5359         return -rte_errno;
5360 }
5361
5362 /**
5363  * Release the flow matcher.
5364  *
5365  * @param dev
5366  *   Pointer to Ethernet device.
5367  * @param flow
5368  *   Pointer to mlx5_flow.
5369  *
5370  * @return
5371  *   1 while a reference on it exists, 0 when freed.
5372  */
5373 static int
5374 flow_dv_matcher_release(struct rte_eth_dev *dev,
5375                         struct mlx5_flow *flow)
5376 {
5377         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
5378         struct mlx5_priv *priv = dev->data->dev_private;
5379         struct mlx5_ibv_shared *sh = priv->sh;
5380         struct mlx5_flow_tbl_resource *tbl;
5381
5382         assert(matcher->matcher_object);
5383         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
5384                 dev->data->port_id, (void *)matcher,
5385                 rte_atomic32_read(&matcher->refcnt));
5386         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
5387                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
5388                            (matcher->matcher_object));
5389                 LIST_REMOVE(matcher, next);
5390                 if (matcher->egress)
5391                         tbl = &sh->tx_tbl[matcher->group];
5392                 else
5393                         tbl = &sh->rx_tbl[matcher->group];
5394                 flow_dv_tbl_resource_release(tbl);
5395                 rte_free(matcher);
5396                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
5397                         dev->data->port_id, (void *)matcher);
5398                 return 0;
5399         }
5400         return 1;
5401 }
5402
5403 /**
5404  * Release an encap/decap resource.
5405  *
5406  * @param flow
5407  *   Pointer to mlx5_flow.
5408  *
5409  * @return
5410  *   1 while a reference on it exists, 0 when freed.
5411  */
5412 static int
5413 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
5414 {
5415         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
5416                                                 flow->dv.encap_decap;
5417
5418         assert(cache_resource->verbs_action);
5419         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
5420                 (void *)cache_resource,
5421                 rte_atomic32_read(&cache_resource->refcnt));
5422         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5423                 claim_zero(mlx5_glue->destroy_flow_action
5424                                 (cache_resource->verbs_action));
5425                 LIST_REMOVE(cache_resource, next);
5426                 rte_free(cache_resource);
5427                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
5428                         (void *)cache_resource);
5429                 return 0;
5430         }
5431         return 1;
5432 }
5433
5434 /**
5435  * Release an jump to table action resource.
5436  *
5437  * @param flow
5438  *   Pointer to mlx5_flow.
5439  *
5440  * @return
5441  *   1 while a reference on it exists, 0 when freed.
5442  */
5443 static int
5444 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
5445 {
5446         struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
5447                                                 flow->dv.jump;
5448
5449         assert(cache_resource->action);
5450         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
5451                 (void *)cache_resource,
5452                 rte_atomic32_read(&cache_resource->refcnt));
5453         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5454                 claim_zero(mlx5_glue->destroy_flow_action
5455                                 (cache_resource->action));
5456                 LIST_REMOVE(cache_resource, next);
5457                 flow_dv_tbl_resource_release(cache_resource->tbl);
5458                 rte_free(cache_resource);
5459                 DRV_LOG(DEBUG, "jump table resource %p: removed",
5460                         (void *)cache_resource);
5461                 return 0;
5462         }
5463         return 1;
5464 }
5465
5466 /**
5467  * Release a modify-header resource.
5468  *
5469  * @param flow
5470  *   Pointer to mlx5_flow.
5471  *
5472  * @return
5473  *   1 while a reference on it exists, 0 when freed.
5474  */
5475 static int
5476 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
5477 {
5478         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
5479                                                 flow->dv.modify_hdr;
5480
5481         assert(cache_resource->verbs_action);
5482         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
5483                 (void *)cache_resource,
5484                 rte_atomic32_read(&cache_resource->refcnt));
5485         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5486                 claim_zero(mlx5_glue->destroy_flow_action
5487                                 (cache_resource->verbs_action));
5488                 LIST_REMOVE(cache_resource, next);
5489                 rte_free(cache_resource);
5490                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
5491                         (void *)cache_resource);
5492                 return 0;
5493         }
5494         return 1;
5495 }
5496
5497 /**
5498  * Release port ID action resource.
5499  *
5500  * @param flow
5501  *   Pointer to mlx5_flow.
5502  *
5503  * @return
5504  *   1 while a reference on it exists, 0 when freed.
5505  */
5506 static int
5507 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
5508 {
5509         struct mlx5_flow_dv_port_id_action_resource *cache_resource =
5510                 flow->dv.port_id_action;
5511
5512         assert(cache_resource->action);
5513         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
5514                 (void *)cache_resource,
5515                 rte_atomic32_read(&cache_resource->refcnt));
5516         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5517                 claim_zero(mlx5_glue->destroy_flow_action
5518                                 (cache_resource->action));
5519                 LIST_REMOVE(cache_resource, next);
5520                 rte_free(cache_resource);
5521                 DRV_LOG(DEBUG, "port id action resource %p: removed",
5522                         (void *)cache_resource);
5523                 return 0;
5524         }
5525         return 1;
5526 }
5527
5528 /**
5529  * Remove the flow from the NIC but keeps it in memory.
5530  *
5531  * @param[in] dev
5532  *   Pointer to Ethernet device.
5533  * @param[in, out] flow
5534  *   Pointer to flow structure.
5535  */
5536 static void
5537 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5538 {
5539         struct mlx5_flow_dv *dv;
5540         struct mlx5_flow *dev_flow;
5541
5542         if (!flow)
5543                 return;
5544         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5545                 dv = &dev_flow->dv;
5546                 if (dv->flow) {
5547                         claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
5548                         dv->flow = NULL;
5549                 }
5550                 if (dv->hrxq) {
5551                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
5552                                 mlx5_hrxq_drop_release(dev);
5553                         else
5554                                 mlx5_hrxq_release(dev, dv->hrxq);
5555                         dv->hrxq = NULL;
5556                 }
5557                 if (dev_flow->dv.vf_vlan.tag &&
5558                     dev_flow->dv.vf_vlan.created)
5559                         mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5560         }
5561 }
5562
5563 /**
5564  * Remove the flow from the NIC and the memory.
5565  *
5566  * @param[in] dev
5567  *   Pointer to the Ethernet device structure.
5568  * @param[in, out] flow
5569  *   Pointer to flow structure.
5570  */
5571 static void
5572 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5573 {
5574         struct mlx5_flow *dev_flow;
5575
5576         if (!flow)
5577                 return;
5578         flow_dv_remove(dev, flow);
5579         if (flow->counter) {
5580                 flow_dv_counter_release(dev, flow->counter);
5581                 flow->counter = NULL;
5582         }
5583         if (flow->tag_resource) {
5584                 flow_dv_tag_release(dev, flow->tag_resource);
5585                 flow->tag_resource = NULL;
5586         }
5587         while (!LIST_EMPTY(&flow->dev_flows)) {
5588                 dev_flow = LIST_FIRST(&flow->dev_flows);
5589                 LIST_REMOVE(dev_flow, next);
5590                 if (dev_flow->dv.matcher)
5591                         flow_dv_matcher_release(dev, dev_flow);
5592                 if (dev_flow->dv.encap_decap)
5593                         flow_dv_encap_decap_resource_release(dev_flow);
5594                 if (dev_flow->dv.modify_hdr)
5595                         flow_dv_modify_hdr_resource_release(dev_flow);
5596                 if (dev_flow->dv.jump)
5597                         flow_dv_jump_tbl_resource_release(dev_flow);
5598                 if (dev_flow->dv.port_id_action)
5599                         flow_dv_port_id_action_resource_release(dev_flow);
5600                 rte_free(dev_flow);
5601         }
5602 }
5603
5604 /**
5605  * Query a dv flow  rule for its statistics via devx.
5606  *
5607  * @param[in] dev
5608  *   Pointer to Ethernet device.
5609  * @param[in] flow
5610  *   Pointer to the sub flow.
5611  * @param[out] data
5612  *   data retrieved by the query.
5613  * @param[out] error
5614  *   Perform verbose error reporting if not NULL.
5615  *
5616  * @return
5617  *   0 on success, a negative errno value otherwise and rte_errno is set.
5618  */
5619 static int
5620 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
5621                     void *data, struct rte_flow_error *error)
5622 {
5623         struct mlx5_priv *priv = dev->data->dev_private;
5624         struct rte_flow_query_count *qc = data;
5625
5626         if (!priv->config.devx)
5627                 return rte_flow_error_set(error, ENOTSUP,
5628                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5629                                           NULL,
5630                                           "counters are not supported");
5631         if (flow->counter) {
5632                 uint64_t pkts, bytes;
5633                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
5634                                                &bytes);
5635
5636                 if (err)
5637                         return rte_flow_error_set(error, -err,
5638                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5639                                         NULL, "cannot read counters");
5640                 qc->hits_set = 1;
5641                 qc->bytes_set = 1;
5642                 qc->hits = pkts - flow->counter->hits;
5643                 qc->bytes = bytes - flow->counter->bytes;
5644                 if (qc->reset) {
5645                         flow->counter->hits = pkts;
5646                         flow->counter->bytes = bytes;
5647                 }
5648                 return 0;
5649         }
5650         return rte_flow_error_set(error, EINVAL,
5651                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5652                                   NULL,
5653                                   "counters are not available");
5654 }
5655
5656 /**
5657  * Query a flow.
5658  *
5659  * @see rte_flow_query()
5660  * @see rte_flow_ops
5661  */
5662 static int
5663 flow_dv_query(struct rte_eth_dev *dev,
5664               struct rte_flow *flow __rte_unused,
5665               const struct rte_flow_action *actions __rte_unused,
5666               void *data __rte_unused,
5667               struct rte_flow_error *error __rte_unused)
5668 {
5669         int ret = -EINVAL;
5670
5671         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5672                 switch (actions->type) {
5673                 case RTE_FLOW_ACTION_TYPE_VOID:
5674                         break;
5675                 case RTE_FLOW_ACTION_TYPE_COUNT:
5676                         ret = flow_dv_query_count(dev, flow, data, error);
5677                         break;
5678                 default:
5679                         return rte_flow_error_set(error, ENOTSUP,
5680                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5681                                                   actions,
5682                                                   "action not supported");
5683                 }
5684         }
5685         return ret;
5686 }
5687
5688 /*
5689  * Mutex-protected thunk to flow_dv_translate().
5690  */
5691 static int
5692 flow_d_translate(struct rte_eth_dev *dev,
5693                  struct mlx5_flow *dev_flow,
5694                  const struct rte_flow_attr *attr,
5695                  const struct rte_flow_item items[],
5696                  const struct rte_flow_action actions[],
5697                  struct rte_flow_error *error)
5698 {
5699         int ret;
5700
5701         flow_d_shared_lock(dev);
5702         ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
5703         flow_d_shared_unlock(dev);
5704         return ret;
5705 }
5706
5707 /*
5708  * Mutex-protected thunk to flow_dv_apply().
5709  */
5710 static int
5711 flow_d_apply(struct rte_eth_dev *dev,
5712              struct rte_flow *flow,
5713              struct rte_flow_error *error)
5714 {
5715         int ret;
5716
5717         flow_d_shared_lock(dev);
5718         ret = flow_dv_apply(dev, flow, error);
5719         flow_d_shared_unlock(dev);
5720         return ret;
5721 }
5722
5723 /*
5724  * Mutex-protected thunk to flow_dv_remove().
5725  */
5726 static void
5727 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5728 {
5729         flow_d_shared_lock(dev);
5730         flow_dv_remove(dev, flow);
5731         flow_d_shared_unlock(dev);
5732 }
5733
5734 /*
5735  * Mutex-protected thunk to flow_dv_destroy().
5736  */
5737 static void
5738 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5739 {
5740         flow_d_shared_lock(dev);
5741         flow_dv_destroy(dev, flow);
5742         flow_d_shared_unlock(dev);
5743 }
5744
5745 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
5746         .validate = flow_dv_validate,
5747         .prepare = flow_dv_prepare,
5748         .translate = flow_d_translate,
5749         .apply = flow_d_apply,
5750         .remove = flow_d_remove,
5751         .destroy = flow_d_destroy,
5752         .query = flow_dv_query,
5753 };
5754
5755 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */