drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  37
  38 #define MLX5_ENCAP_MAX_LEN 132
  39
  40 /**
  41  * Validate META item.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the rte_eth_dev structure.
  45  * @param[in] item
  46  *   Item specification.
  47  * @param[in] attr
  48  *   Attributes of flow that includes this item.
  49  * @param[out] error
  50  *   Pointer to error structure.
  51  *
  52  * @return
  53  *   0 on success, a negative errno value otherwise and rte_errno is set.
  54  */
  55 static int
  56 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
  57                            const struct rte_flow_item *item,
  58                            const struct rte_flow_attr *attr,
  59                            struct rte_flow_error *error)
  60 {
  61         const struct rte_flow_item_meta *spec = item->spec;
  62         const struct rte_flow_item_meta *mask = item->mask;
  63         const struct rte_flow_item_meta nic_mask = {
  64                 .data = RTE_BE32(UINT32_MAX)
  65         };
  66         int ret;
  67         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
  68
  69         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
  70                 return rte_flow_error_set(error, EPERM,
  71                                           RTE_FLOW_ERROR_TYPE_ITEM,
  72                                           NULL,
  73                                           "match on metadata offload "
  74                                           "configuration is off for this port");
  75         if (!spec)
  76                 return rte_flow_error_set(error, EINVAL,
  77                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  78                                           item->spec,
  79                                           "data cannot be empty");
  80         if (!spec->data)
  81                 return rte_flow_error_set(error, EINVAL,
  82                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  83                                           NULL,
  84                                           "data cannot be zero");
  85         if (!mask)
  86                 mask = &rte_flow_item_meta_mask;
  87         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
  88                                         (const uint8_t *)&nic_mask,
  89                                         sizeof(struct rte_flow_item_meta),
  90                                         error);
  91         if (ret < 0)
  92                 return ret;
  93         if (attr->ingress)
  94                 return rte_flow_error_set(error, ENOTSUP,
  95                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
  96                                           NULL,
  97                                           "pattern not supported for ingress");
  98         return 0;
  99 }
 100
 101 /**
 102  * Validate the L2 encap action.
 103  *
 104  * @param[in] action_flags
 105  *   Holds the actions detected until now.
 106  * @param[in] action
 107  *   Pointer to the encap action.
 108  * @param[in] attr
 109  *   Pointer to flow attributes
 110  * @param[out] error
 111  *   Pointer to error structure.
 112  *
 113  * @return
 114  *   0 on success, a negative errno value otherwise and rte_errno is set.
 115  */
 116 static int
 117 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 118                                  const struct rte_flow_action *action,
 119                                  const struct rte_flow_attr *attr,
 120                                  struct rte_flow_error *error)
 121 {
 122         if (!(action->conf))
 123                 return rte_flow_error_set(error, EINVAL,
 124                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 125                                           "configuration cannot be null");
 126         if (action_flags & MLX5_FLOW_ACTION_DROP)
 127                 return rte_flow_error_set(error, EINVAL,
 128                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 129                                           "can't drop and encap in same flow");
 130         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 131                 return rte_flow_error_set(error, EINVAL,
 132                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 133                                           "can only have a single encap or"
 134                                           " decap action in a flow");
 135         if (attr->ingress)
 136                 return rte_flow_error_set(error, ENOTSUP,
 137                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 138                                           NULL,
 139                                           "encap action not supported for "
 140                                           "ingress");
 141         return 0;
 142 }
 143
 144 /**
 145  * Validate the L2 decap action.
 146  *
 147  * @param[in] action_flags
 148  *   Holds the actions detected until now.
 149  * @param[in] attr
 150  *   Pointer to flow attributes
 151  * @param[out] error
 152  *   Pointer to error structure.
 153  *
 154  * @return
 155  *   0 on success, a negative errno value otherwise and rte_errno is set.
 156  */
 157 static int
 158 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 159                                  const struct rte_flow_attr *attr,
 160                                  struct rte_flow_error *error)
 161 {
 162         if (action_flags & MLX5_FLOW_ACTION_DROP)
 163                 return rte_flow_error_set(error, EINVAL,
 164                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 165                                           "can't drop and decap in same flow");
 166         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 167                 return rte_flow_error_set(error, EINVAL,
 168                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 169                                           "can only have a single encap or"
 170                                           " decap action in a flow");
 171         if (attr->egress)
 172                 return rte_flow_error_set(error, ENOTSUP,
 173                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 174                                           NULL,
 175                                           "decap action not supported for "
 176                                           "egress");
 177         return 0;
 178 }
 179
 180 /**
 181  * Validate the raw encap action.
 182  *
 183  * @param[in] action_flags
 184  *   Holds the actions detected until now.
 185  * @param[in] action
 186  *   Pointer to the encap action.
 187  * @param[in] attr
 188  *   Pointer to flow attributes
 189  * @param[out] error
 190  *   Pointer to error structure.
 191  *
 192  * @return
 193  *   0 on success, a negative errno value otherwise and rte_errno is set.
 194  */
 195 static int
 196 flow_dv_validate_action_raw_encap(uint64_t action_flags,
 197                                   const struct rte_flow_action *action,
 198                                   const struct rte_flow_attr *attr,
 199                                   struct rte_flow_error *error)
 200 {
 201         if (!(action->conf))
 202                 return rte_flow_error_set(error, EINVAL,
 203                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 204                                           "configuration cannot be null");
 205         if (action_flags & MLX5_FLOW_ACTION_DROP)
 206                 return rte_flow_error_set(error, EINVAL,
 207                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 208                                           "can't drop and encap in same flow");
 209         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 210                 return rte_flow_error_set(error, EINVAL,
 211                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 212                                           "can only have a single encap"
 213                                           " action in a flow");
 214         /* encap without preceding decap is not supported for ingress */
 215         if (attr->ingress && !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
 216                 return rte_flow_error_set(error, ENOTSUP,
 217                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 218                                           NULL,
 219                                           "encap action not supported for "
 220                                           "ingress");
 221         return 0;
 222 }
 223
 224 /**
 225  * Validate the raw decap action.
 226  *
 227  * @param[in] action_flags
 228  *   Holds the actions detected until now.
 229  * @param[in] action
 230  *   Pointer to the encap action.
 231  * @param[in] attr
 232  *   Pointer to flow attributes
 233  * @param[out] error
 234  *   Pointer to error structure.
 235  *
 236  * @return
 237  *   0 on success, a negative errno value otherwise and rte_errno is set.
 238  */
 239 static int
 240 flow_dv_validate_action_raw_decap(uint64_t action_flags,
 241                                   const struct rte_flow_action *action,
 242                                   const struct rte_flow_attr *attr,
 243                                   struct rte_flow_error *error)
 244 {
 245         if (action_flags & MLX5_FLOW_ACTION_DROP)
 246                 return rte_flow_error_set(error, EINVAL,
 247                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 248                                           "can't drop and decap in same flow");
 249         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 250                 return rte_flow_error_set(error, EINVAL,
 251                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 252                                           "can't have encap action before"
 253                                           " decap action");
 254         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
 255                 return rte_flow_error_set(error, EINVAL,
 256                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 257                                           "can only have a single decap"
 258                                           " action in a flow");
 259         /* decap action is valid on egress only if it is followed by encap */
 260         if (attr->egress) {
 261                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
 262                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
 263                        action++) {
 264                 }
 265                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
 266                         return rte_flow_error_set
 267                                         (error, ENOTSUP,
 268                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 269                                          NULL, "decap action not supported"
 270                                          " for egress");
 271         }
 272         return 0;
 273 }
 274
 275 /**
 276  * Get the size of specific rte_flow_item_type
 277  *
 278  * @param[in] item_type
 279  *   Tested rte_flow_item_type.
 280  *
 281  * @return
 282  *   sizeof struct item_type, 0 if void or irrelevant.
 283  */
 284 static size_t
 285 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
 286 {
 287         size_t retval;
 288
 289         switch (item_type) {
 290         case RTE_FLOW_ITEM_TYPE_ETH:
 291                 retval = sizeof(struct rte_flow_item_eth);
 292                 break;
 293         case RTE_FLOW_ITEM_TYPE_VLAN:
 294                 retval = sizeof(struct rte_flow_item_vlan);
 295                 break;
 296         case RTE_FLOW_ITEM_TYPE_IPV4:
 297                 retval = sizeof(struct rte_flow_item_ipv4);
 298                 break;
 299         case RTE_FLOW_ITEM_TYPE_IPV6:
 300                 retval = sizeof(struct rte_flow_item_ipv6);
 301                 break;
 302         case RTE_FLOW_ITEM_TYPE_UDP:
 303                 retval = sizeof(struct rte_flow_item_udp);
 304                 break;
 305         case RTE_FLOW_ITEM_TYPE_TCP:
 306                 retval = sizeof(struct rte_flow_item_tcp);
 307                 break;
 308         case RTE_FLOW_ITEM_TYPE_VXLAN:
 309                 retval = sizeof(struct rte_flow_item_vxlan);
 310                 break;
 311         case RTE_FLOW_ITEM_TYPE_GRE:
 312                 retval = sizeof(struct rte_flow_item_gre);
 313                 break;
 314         case RTE_FLOW_ITEM_TYPE_NVGRE:
 315                 retval = sizeof(struct rte_flow_item_nvgre);
 316                 break;
 317         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 318                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
 319                 break;
 320         case RTE_FLOW_ITEM_TYPE_MPLS:
 321                 retval = sizeof(struct rte_flow_item_mpls);
 322                 break;
 323         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
 324         default:
 325                 retval = 0;
 326                 break;
 327         }
 328         return retval;
 329 }
 330
 331 #define MLX5_ENCAP_IPV4_VERSION         0x40
 332 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
 333 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
 334 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
 335 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
 336 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
 337 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
 338
 339 /**
 340  * Convert the encap action data from list of rte_flow_item to raw buffer
 341  *
 342  * @param[in] items
 343  *   Pointer to rte_flow_item objects list.
 344  * @param[out] buf
 345  *   Pointer to the output buffer.
 346  * @param[out] size
 347  *   Pointer to the output buffer size.
 348  * @param[out] error
 349  *   Pointer to the error structure.
 350  *
 351  * @return
 352  *   0 on success, a negative errno value otherwise and rte_errno is set.
 353  */
 354 static int
 355 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 356                            size_t *size, struct rte_flow_error *error)
 357 {
 358         struct ether_hdr *eth = NULL;
 359         struct vlan_hdr *vlan = NULL;
 360         struct ipv4_hdr *ipv4 = NULL;
 361         struct ipv6_hdr *ipv6 = NULL;
 362         struct udp_hdr *udp = NULL;
 363         struct vxlan_hdr *vxlan = NULL;
 364         struct vxlan_gpe_hdr *vxlan_gpe = NULL;
 365         struct gre_hdr *gre = NULL;
 366         size_t len;
 367         size_t temp_size = 0;
 368
 369         if (!items)
 370                 return rte_flow_error_set(error, EINVAL,
 371                                           RTE_FLOW_ERROR_TYPE_ACTION,
 372                                           NULL, "invalid empty data");
 373         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 374                 len = flow_dv_get_item_len(items->type);
 375                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
 376                         return rte_flow_error_set(error, EINVAL,
 377                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 378                                                   (void *)items->type,
 379                                                   "items total size is too big"
 380                                                   " for encap action");
 381                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
 382                 switch (items->type) {
 383                 case RTE_FLOW_ITEM_TYPE_ETH:
 384                         eth = (struct ether_hdr *)&buf[temp_size];
 385                         break;
 386                 case RTE_FLOW_ITEM_TYPE_VLAN:
 387                         vlan = (struct vlan_hdr *)&buf[temp_size];
 388                         if (!eth)
 389                                 return rte_flow_error_set(error, EINVAL,
 390                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 391                                                 (void *)items->type,
 392                                                 "eth header not found");
 393                         if (!eth->ether_type)
 394                                 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
 395                         break;
 396                 case RTE_FLOW_ITEM_TYPE_IPV4:
 397                         ipv4 = (struct ipv4_hdr *)&buf[temp_size];
 398                         if (!vlan && !eth)
 399                                 return rte_flow_error_set(error, EINVAL,
 400                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 401                                                 (void *)items->type,
 402                                                 "neither eth nor vlan"
 403                                                 " header found");
 404                         if (vlan && !vlan->eth_proto)
 405                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
 406                         else if (eth && !eth->ether_type)
 407                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
 408                         if (!ipv4->version_ihl)
 409                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
 410                                                     MLX5_ENCAP_IPV4_IHL_MIN;
 411                         if (!ipv4->time_to_live)
 412                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
 413                         break;
 414                 case RTE_FLOW_ITEM_TYPE_IPV6:
 415                         ipv6 = (struct ipv6_hdr *)&buf[temp_size];
 416                         if (!vlan && !eth)
 417                                 return rte_flow_error_set(error, EINVAL,
 418                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 419                                                 (void *)items->type,
 420                                                 "neither eth nor vlan"
 421                                                 " header found");
 422                         if (vlan && !vlan->eth_proto)
 423                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
 424                         else if (eth && !eth->ether_type)
 425                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
 426                         if (!ipv6->vtc_flow)
 427                                 ipv6->vtc_flow =
 428                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
 429                         if (!ipv6->hop_limits)
 430                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
 431                         break;
 432                 case RTE_FLOW_ITEM_TYPE_UDP:
 433                         udp = (struct udp_hdr *)&buf[temp_size];
 434                         if (!ipv4 && !ipv6)
 435                                 return rte_flow_error_set(error, EINVAL,
 436                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 437                                                 (void *)items->type,
 438                                                 "ip header not found");
 439                         if (ipv4 && !ipv4->next_proto_id)
 440                                 ipv4->next_proto_id = IPPROTO_UDP;
 441                         else if (ipv6 && !ipv6->proto)
 442                                 ipv6->proto = IPPROTO_UDP;
 443                         break;
 444                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 445                         vxlan = (struct vxlan_hdr *)&buf[temp_size];
 446                         if (!udp)
 447                                 return rte_flow_error_set(error, EINVAL,
 448                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 449                                                 (void *)items->type,
 450                                                 "udp header not found");
 451                         if (!udp->dst_port)
 452                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
 453                         if (!vxlan->vx_flags)
 454                                 vxlan->vx_flags =
 455                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
 456                         break;
 457                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 458                         vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
 459                         if (!udp)
 460                                 return rte_flow_error_set(error, EINVAL,
 461                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 462                                                 (void *)items->type,
 463                                                 "udp header not found");
 464                         if (!vxlan_gpe->proto)
 465                                 return rte_flow_error_set(error, EINVAL,
 466                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 467                                                 (void *)items->type,
 468                                                 "next protocol not found");
 469                         if (!udp->dst_port)
 470                                 udp->dst_port =
 471                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
 472                         if (!vxlan_gpe->vx_flags)
 473                                 vxlan_gpe->vx_flags =
 474                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
 475                         break;
 476                 case RTE_FLOW_ITEM_TYPE_GRE:
 477                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 478                         gre = (struct gre_hdr *)&buf[temp_size];
 479                         if (!gre->proto)
 480                                 return rte_flow_error_set(error, EINVAL,
 481                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 482                                                 (void *)items->type,
 483                                                 "next protocol not found");
 484                         if (!ipv4 && !ipv6)
 485                                 return rte_flow_error_set(error, EINVAL,
 486                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 487                                                 (void *)items->type,
 488                                                 "ip header not found");
 489                         if (ipv4 && !ipv4->next_proto_id)
 490                                 ipv4->next_proto_id = IPPROTO_GRE;
 491                         else if (ipv6 && !ipv6->proto)
 492                                 ipv6->proto = IPPROTO_GRE;
 493                         break;
 494                 case RTE_FLOW_ITEM_TYPE_VOID:
 495                         break;
 496                 default:
 497                         return rte_flow_error_set(error, EINVAL,
 498                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 499                                                   (void *)items->type,
 500                                                   "unsupported item type");
 501                         break;
 502                 }
 503                 temp_size += len;
 504         }
 505         *size = temp_size;
 506         return 0;
 507 }
 508
 509 /**
 510  * Convert L2 encap action to DV specification.
 511  *
 512  * @param[in] dev
 513  *   Pointer to rte_eth_dev structure.
 514  * @param[in] action
 515  *   Pointer to action structure.
 516  * @param[out] error
 517  *   Pointer to the error structure.
 518  *
 519  * @return
 520  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 521  */
 522 static struct ibv_flow_action *
 523 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
 524                                const struct rte_flow_action *action,
 525                                struct rte_flow_error *error)
 526 {
 527         struct ibv_flow_action *verbs_action = NULL;
 528         const struct rte_flow_item *encap_data;
 529         const struct rte_flow_action_raw_encap *raw_encap_data;
 530         struct priv *priv = dev->data->dev_private;
 531         uint8_t buf[MLX5_ENCAP_MAX_LEN];
 532         uint8_t *buf_ptr = buf;
 533         size_t size = 0;
 534         int convert_result = 0;
 535
 536         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
 537                 raw_encap_data =
 538                         (const struct rte_flow_action_raw_encap *)action->conf;
 539                 buf_ptr = raw_encap_data->data;
 540                 size = raw_encap_data->size;
 541         } else {
 542                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
 543                         encap_data =
 544                                 ((const struct rte_flow_action_vxlan_encap *)
 545                                                 action->conf)->definition;
 546                 else
 547                         encap_data =
 548                                 ((const struct rte_flow_action_nvgre_encap *)
 549                                                 action->conf)->definition;
 550                 convert_result = flow_dv_convert_encap_data(encap_data, buf,
 551                                                             &size, error);
 552                 if (convert_result)
 553                         return NULL;
 554         }
 555         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 556                 (priv->ctx, size, buf_ptr,
 557                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
 558                  MLX5DV_FLOW_TABLE_TYPE_NIC_TX);
 559         if (!verbs_action)
 560                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 561                                    NULL, "cannot create L2 encap action");
 562         return verbs_action;
 563 }
 564
 565 /**
 566  * Convert L2 decap action to DV specification.
 567  *
 568  * @param[in] dev
 569  *   Pointer to rte_eth_dev structure.
 570  * @param[out] error
 571  *   Pointer to the error structure.
 572  *
 573  * @return
 574  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 575  */
 576 static struct ibv_flow_action *
 577 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
 578                                struct rte_flow_error *error)
 579 {
 580         struct ibv_flow_action *verbs_action = NULL;
 581         struct priv *priv = dev->data->dev_private;
 582
 583         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 584                 (priv->ctx, 0, NULL,
 585                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
 586                  MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
 587         if (!verbs_action)
 588                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 589                                    NULL, "cannot create L2 decap action");
 590         return verbs_action;
 591 }
 592
 593 /**
 594  * Convert raw decap/encap (L3 tunnel) action to DV specification.
 595  *
 596  * @param[in] dev
 597  *   Pointer to rte_eth_dev structure.
 598  * @param[in] action
 599  *   Pointer to action structure.
 600  * @param[in] attr
 601  *   Pointer to the flow attributes.
 602  * @param[out] error
 603  *   Pointer to the error structure.
 604  *
 605  * @return
 606  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 607  */
 608 static struct ibv_flow_action *
 609 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
 610                                 const struct rte_flow_action *action,
 611                                 const struct rte_flow_attr *attr,
 612                                 struct rte_flow_error *error)
 613 {
 614         struct ibv_flow_action *verbs_action = NULL;
 615         const struct rte_flow_action_raw_encap *encap_data;
 616         struct priv *priv = dev->data->dev_private;
 617         enum mlx5dv_flow_action_packet_reformat_type reformat_type;
 618         enum mlx5dv_flow_table_type ft_type;
 619
 620         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
 621         reformat_type = attr->egress ?
 622                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
 623                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
 624         ft_type = attr->egress ?
 625                         MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
 626                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
 627         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 628                                 (priv->ctx, encap_data->size,
 629                                 (encap_data->size ? encap_data->data : NULL),
 630                                 reformat_type, ft_type);
 631         if (!verbs_action)
 632                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 633                                    NULL, "cannot create encap action");
 634         return verbs_action;
 635 }
 636
 637 /**
 638  * Verify the @p attributes will be correctly understood by the NIC and store
 639  * them in the @p flow if everything is correct.
 640  *
 641  * @param[in] dev
 642  *   Pointer to dev struct.
 643  * @param[in] attributes
 644  *   Pointer to flow attributes
 645  * @param[out] error
 646  *   Pointer to error structure.
 647  *
 648  * @return
 649  *   0 on success, a negative errno value otherwise and rte_errno is set.
 650  */
 651 static int
 652 flow_dv_validate_attributes(struct rte_eth_dev *dev,
 653                             const struct rte_flow_attr *attributes,
 654                             struct rte_flow_error *error)
 655 {
 656         struct priv *priv = dev->data->dev_private;
 657         uint32_t priority_max = priv->config.flow_prio - 1;
 658
 659         if (attributes->group)
 660                 return rte_flow_error_set(error, ENOTSUP,
 661                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 662                                           NULL,
 663                                           "groups is not supported");
 664         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
 665             attributes->priority >= priority_max)
 666                 return rte_flow_error_set(error, ENOTSUP,
 667                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 668                                           NULL,
 669                                           "priority out of range");
 670         if (attributes->transfer)
 671                 return rte_flow_error_set(error, ENOTSUP,
 672                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
 673                                           NULL,
 674                                           "transfer is not supported");
 675         if (!(attributes->egress ^ attributes->ingress))
 676                 return rte_flow_error_set(error, ENOTSUP,
 677                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 678                                           "must specify exactly one of "
 679                                           "ingress or egress");
 680         return 0;
 681 }
 682
 683 /**
 684  * Internal validation function. For validating both actions and items.
 685  *
 686  * @param[in] dev
 687  *   Pointer to the rte_eth_dev structure.
 688  * @param[in] attr
 689  *   Pointer to the flow attributes.
 690  * @param[in] items
 691  *   Pointer to the list of items.
 692  * @param[in] actions
 693  *   Pointer to the list of actions.
 694  * @param[out] error
 695  *   Pointer to the error structure.
 696  *
 697  * @return
 698  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 699  */
 700 static int
 701 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 702                  const struct rte_flow_item items[],
 703                  const struct rte_flow_action actions[],
 704                  struct rte_flow_error *error)
 705 {
 706         int ret;
 707         uint64_t action_flags = 0;
 708         uint64_t item_flags = 0;
 709         int tunnel = 0;
 710         uint8_t next_protocol = 0xff;
 711         int actions_n = 0;
 712
 713         if (items == NULL)
 714                 return -1;
 715         ret = flow_dv_validate_attributes(dev, attr, error);
 716         if (ret < 0)
 717                 return ret;
 718         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 719                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 720                 switch (items->type) {
 721                 case RTE_FLOW_ITEM_TYPE_VOID:
 722                         break;
 723                 case RTE_FLOW_ITEM_TYPE_ETH:
 724                         ret = mlx5_flow_validate_item_eth(items, item_flags,
 725                                                           error);
 726                         if (ret < 0)
 727                                 return ret;
 728                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 729                                                MLX5_FLOW_LAYER_OUTER_L2;
 730                         break;
 731                 case RTE_FLOW_ITEM_TYPE_VLAN:
 732                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
 733                                                            error);
 734                         if (ret < 0)
 735                                 return ret;
 736                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
 737                                                MLX5_FLOW_LAYER_OUTER_VLAN;
 738                         break;
 739                 case RTE_FLOW_ITEM_TYPE_IPV4:
 740                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
 741                                                            error);
 742                         if (ret < 0)
 743                                 return ret;
 744                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 745                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 746                         if (items->mask != NULL &&
 747                             ((const struct rte_flow_item_ipv4 *)
 748                              items->mask)->hdr.next_proto_id)
 749                                 next_protocol =
 750                                         ((const struct rte_flow_item_ipv4 *)
 751                                          (items->spec))->hdr.next_proto_id;
 752                         break;
 753                 case RTE_FLOW_ITEM_TYPE_IPV6:
 754                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
 755                                                            error);
 756                         if (ret < 0)
 757                                 return ret;
 758                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 759                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 760                         if (items->mask != NULL &&
 761                             ((const struct rte_flow_item_ipv6 *)
 762                              items->mask)->hdr.proto)
 763                                 next_protocol =
 764                                         ((const struct rte_flow_item_ipv6 *)
 765                                          items->spec)->hdr.proto;
 766                         break;
 767                 case RTE_FLOW_ITEM_TYPE_TCP:
 768                         ret = mlx5_flow_validate_item_tcp
 769                                                 (items, item_flags,
 770                                                  next_protocol,
 771                                                  &rte_flow_item_tcp_mask,
 772                                                  error);
 773                         if (ret < 0)
 774                                 return ret;
 775                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 776                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
 777                         break;
 778                 case RTE_FLOW_ITEM_TYPE_UDP:
 779                         ret = mlx5_flow_validate_item_udp(items, item_flags,
 780                                                           next_protocol,
 781                                                           error);
 782                         if (ret < 0)
 783                                 return ret;
 784                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 785                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
 786                         break;
 787                 case RTE_FLOW_ITEM_TYPE_GRE:
 788                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 789                         ret = mlx5_flow_validate_item_gre(items, item_flags,
 790                                                           next_protocol, error);
 791                         if (ret < 0)
 792                                 return ret;
 793                         item_flags |= MLX5_FLOW_LAYER_GRE;
 794                         break;
 795                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 796                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
 797                                                             error);
 798                         if (ret < 0)
 799                                 return ret;
 800                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
 801                         break;
 802                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 803                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
 804                                                                 item_flags, dev,
 805                                                                 error);
 806                         if (ret < 0)
 807                                 return ret;
 808                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 809                         break;
 810                 case RTE_FLOW_ITEM_TYPE_META:
 811                         ret = flow_dv_validate_item_meta(dev, items, attr,
 812                                                          error);
 813                         if (ret < 0)
 814                                 return ret;
 815                         item_flags |= MLX5_FLOW_ITEM_METADATA;
 816                         break;
 817                 default:
 818                         return rte_flow_error_set(error, ENOTSUP,
 819                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 820                                                   NULL, "item not supported");
 821                 }
 822         }
 823         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 824                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
 825                         return rte_flow_error_set(error, ENOTSUP,
 826                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 827                                                   actions, "too many actions");
 828                 switch (actions->type) {
 829                 case RTE_FLOW_ACTION_TYPE_VOID:
 830                         break;
 831                 case RTE_FLOW_ACTION_TYPE_FLAG:
 832                         ret = mlx5_flow_validate_action_flag(action_flags,
 833                                                              attr, error);
 834                         if (ret < 0)
 835                                 return ret;
 836                         action_flags |= MLX5_FLOW_ACTION_FLAG;
 837                         ++actions_n;
 838                         break;
 839                 case RTE_FLOW_ACTION_TYPE_MARK:
 840                         ret = mlx5_flow_validate_action_mark(actions,
 841                                                              action_flags,
 842                                                              attr, error);
 843                         if (ret < 0)
 844                                 return ret;
 845                         action_flags |= MLX5_FLOW_ACTION_MARK;
 846                         ++actions_n;
 847                         break;
 848                 case RTE_FLOW_ACTION_TYPE_DROP:
 849                         ret = mlx5_flow_validate_action_drop(action_flags,
 850                                                              attr, error);
 851                         if (ret < 0)
 852                                 return ret;
 853                         action_flags |= MLX5_FLOW_ACTION_DROP;
 854                         ++actions_n;
 855                         break;
 856                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 857                         ret = mlx5_flow_validate_action_queue(actions,
 858                                                               action_flags, dev,
 859                                                               attr, error);
 860                         if (ret < 0)
 861                                 return ret;
 862                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
 863                         ++actions_n;
 864                         break;
 865                 case RTE_FLOW_ACTION_TYPE_RSS:
 866                         ret = mlx5_flow_validate_action_rss(actions,
 867                                                             action_flags, dev,
 868                                                             attr, error);
 869                         if (ret < 0)
 870                                 return ret;
 871                         action_flags |= MLX5_FLOW_ACTION_RSS;
 872                         ++actions_n;
 873                         break;
 874                 case RTE_FLOW_ACTION_TYPE_COUNT:
 875                         ret = mlx5_flow_validate_action_count(dev, attr, error);
 876                         if (ret < 0)
 877                                 return ret;
 878                         action_flags |= MLX5_FLOW_ACTION_COUNT;
 879                         ++actions_n;
 880                         break;
 881                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 882                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 883                         ret = flow_dv_validate_action_l2_encap(action_flags,
 884                                                                actions, attr,
 885                                                                error);
 886                         if (ret < 0)
 887                                 return ret;
 888                         action_flags |= actions->type ==
 889                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
 890                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
 891                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
 892                         ++actions_n;
 893                         break;
 894                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
 895                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
 896                         ret = flow_dv_validate_action_l2_decap(action_flags,
 897                                                                attr, error);
 898                         if (ret < 0)
 899                                 return ret;
 900                         action_flags |= actions->type ==
 901                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
 902                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
 903                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
 904                         ++actions_n;
 905                         break;
 906                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
 907                         ret = flow_dv_validate_action_raw_encap(action_flags,
 908                                                                 actions, attr,
 909                                                                 error);
 910                         if (ret < 0)
 911                                 return ret;
 912                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
 913                         ++actions_n;
 914                         break;
 915                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
 916                         ret = flow_dv_validate_action_raw_decap(action_flags,
 917                                                                 actions, attr,
 918                                                                 error);
 919                         if (ret < 0)
 920                                 return ret;
 921                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
 922                         ++actions_n;
 923                         break;
 924                 default:
 925                         return rte_flow_error_set(error, ENOTSUP,
 926                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 927                                                   actions,
 928                                                   "action not supported");
 929                 }
 930         }
 931         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
 932                 return rte_flow_error_set(error, EINVAL,
 933                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
 934                                           "no fate action is found");
 935         return 0;
 936 }
 937
 938 /**
 939  * Internal preparation function. Allocates the DV flow size,
 940  * this size is constant.
 941  *
 942  * @param[in] attr
 943  *   Pointer to the flow attributes.
 944  * @param[in] items
 945  *   Pointer to the list of items.
 946  * @param[in] actions
 947  *   Pointer to the list of actions.
 948  * @param[out] item_flags
 949  *   Pointer to bit mask of all items detected.
 950  * @param[out] action_flags
 951  *   Pointer to bit mask of all actions detected.
 952  * @param[out] error
 953  *   Pointer to the error structure.
 954  *
 955  * @return
 956  *   Pointer to mlx5_flow object on success,
 957  *   otherwise NULL and rte_ernno is set.
 958  */
 959 static struct mlx5_flow *
 960 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
 961                 const struct rte_flow_item items[] __rte_unused,
 962                 const struct rte_flow_action actions[] __rte_unused,
 963                 uint64_t *item_flags __rte_unused,
 964                 uint64_t *action_flags __rte_unused,
 965                 struct rte_flow_error *error)
 966 {
 967         uint32_t size = sizeof(struct mlx5_flow);
 968         struct mlx5_flow *flow;
 969
 970         flow = rte_calloc(__func__, 1, size, 0);
 971         if (!flow) {
 972                 rte_flow_error_set(error, ENOMEM,
 973                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 974                                    "not enough memory to create flow");
 975                 return NULL;
 976         }
 977         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
 978         return flow;
 979 }
 980
 981 /**
 982  * Add Ethernet item to matcher and to the value.
 983  *
 984  * @param[in, out] matcher
 985  *   Flow matcher.
 986  * @param[in, out] key
 987  *   Flow matcher value.
 988  * @param[in] item
 989  *   Flow pattern to translate.
 990  * @param[in] inner
 991  *   Item is inner pattern.
 992  */
 993 static void
 994 flow_dv_translate_item_eth(void *matcher, void *key,
 995                            const struct rte_flow_item *item, int inner)
 996 {
 997         const struct rte_flow_item_eth *eth_m = item->mask;
 998         const struct rte_flow_item_eth *eth_v = item->spec;
 999         const struct rte_flow_item_eth nic_mask = {
1000                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1001                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1002                 .type = RTE_BE16(0xffff),
1003         };
1004         void *headers_m;
1005         void *headers_v;
1006         char *l24_v;
1007         unsigned int i;
1008
1009         if (!eth_v)
1010                 return;
1011         if (!eth_m)
1012                 eth_m = &nic_mask;
1013         if (inner) {
1014                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1015                                          inner_headers);
1016                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1017         } else {
1018                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1019                                          outer_headers);
1020                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1021         }
1022         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
1023                &eth_m->dst, sizeof(eth_m->dst));
1024         /* The value must be in the range of the mask. */
1025         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
1026         for (i = 0; i < sizeof(eth_m->dst); ++i)
1027                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
1028         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
1029                &eth_m->src, sizeof(eth_m->src));
1030         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
1031         /* The value must be in the range of the mask. */
1032         for (i = 0; i < sizeof(eth_m->dst); ++i)
1033                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
1034         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
1035                  rte_be_to_cpu_16(eth_m->type));
1036         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
1037         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
1038 }
1039
1040 /**
1041  * Add VLAN item to matcher and to the value.
1042  *
1043  * @param[in, out] matcher
1044  *   Flow matcher.
1045  * @param[in, out] key
1046  *   Flow matcher value.
1047  * @param[in] item
1048  *   Flow pattern to translate.
1049  * @param[in] inner
1050  *   Item is inner pattern.
1051  */
1052 static void
1053 flow_dv_translate_item_vlan(void *matcher, void *key,
1054                             const struct rte_flow_item *item,
1055                             int inner)
1056 {
1057         const struct rte_flow_item_vlan *vlan_m = item->mask;
1058         const struct rte_flow_item_vlan *vlan_v = item->spec;
1059         const struct rte_flow_item_vlan nic_mask = {
1060                 .tci = RTE_BE16(0x0fff),
1061                 .inner_type = RTE_BE16(0xffff),
1062         };
1063         void *headers_m;
1064         void *headers_v;
1065         uint16_t tci_m;
1066         uint16_t tci_v;
1067
1068         if (!vlan_v)
1069                 return;
1070         if (!vlan_m)
1071                 vlan_m = &nic_mask;
1072         if (inner) {
1073                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1074                                          inner_headers);
1075                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1076         } else {
1077                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1078                                          outer_headers);
1079                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1080         }
1081         tci_m = rte_be_to_cpu_16(vlan_m->tci);
1082         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
1083         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
1084         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
1085         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
1086         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
1087         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
1088         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
1089         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
1090         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
1091 }
1092
1093 /**
1094  * Add IPV4 item to matcher and to the value.
1095  *
1096  * @param[in, out] matcher
1097  *   Flow matcher.
1098  * @param[in, out] key
1099  *   Flow matcher value.
1100  * @param[in] item
1101  *   Flow pattern to translate.
1102  * @param[in] inner
1103  *   Item is inner pattern.
1104  */
1105 static void
1106 flow_dv_translate_item_ipv4(void *matcher, void *key,
1107                             const struct rte_flow_item *item,
1108                             int inner)
1109 {
1110         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
1111         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
1112         const struct rte_flow_item_ipv4 nic_mask = {
1113                 .hdr = {
1114                         .src_addr = RTE_BE32(0xffffffff),
1115                         .dst_addr = RTE_BE32(0xffffffff),
1116                         .type_of_service = 0xff,
1117                         .next_proto_id = 0xff,
1118                 },
1119         };
1120         void *headers_m;
1121         void *headers_v;
1122         char *l24_m;
1123         char *l24_v;
1124         uint8_t tos;
1125
1126         if (inner) {
1127                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1128                                          inner_headers);
1129                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1130         } else {
1131                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1132                                          outer_headers);
1133                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1134         }
1135         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
1136         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
1137         if (!ipv4_v)
1138                 return;
1139         if (!ipv4_m)
1140                 ipv4_m = &nic_mask;
1141         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1142                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
1143         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1144                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
1145         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
1146         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
1147         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1148                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
1149         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1150                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
1151         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
1152         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
1153         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
1154         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
1155                  ipv4_m->hdr.type_of_service);
1156         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
1157         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
1158                  ipv4_m->hdr.type_of_service >> 2);
1159         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
1160         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
1161                  ipv4_m->hdr.next_proto_id);
1162         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1163                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
1164 }
1165
1166 /**
1167  * Add IPV6 item to matcher and to the value.
1168  *
1169  * @param[in, out] matcher
1170  *   Flow matcher.
1171  * @param[in, out] key
1172  *   Flow matcher value.
1173  * @param[in] item
1174  *   Flow pattern to translate.
1175  * @param[in] inner
1176  *   Item is inner pattern.
1177  */
1178 static void
1179 flow_dv_translate_item_ipv6(void *matcher, void *key,
1180                             const struct rte_flow_item *item,
1181                             int inner)
1182 {
1183         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
1184         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
1185         const struct rte_flow_item_ipv6 nic_mask = {
1186                 .hdr = {
1187                         .src_addr =
1188                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1189                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1190                         .dst_addr =
1191                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1192                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1193                         .vtc_flow = RTE_BE32(0xffffffff),
1194                         .proto = 0xff,
1195                         .hop_limits = 0xff,
1196                 },
1197         };
1198         void *headers_m;
1199         void *headers_v;
1200         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1201         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1202         char *l24_m;
1203         char *l24_v;
1204         uint32_t vtc_m;
1205         uint32_t vtc_v;
1206         int i;
1207         int size;
1208
1209         if (inner) {
1210                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1211                                          inner_headers);
1212                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1213         } else {
1214                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1215                                          outer_headers);
1216                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1217         }
1218         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
1219         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
1220         if (!ipv6_v)
1221                 return;
1222         if (!ipv6_m)
1223                 ipv6_m = &nic_mask;
1224         size = sizeof(ipv6_m->hdr.dst_addr);
1225         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1226                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1227         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1228                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1229         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
1230         for (i = 0; i < size; ++i)
1231                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
1232         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1233                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1234         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1235                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1236         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
1237         for (i = 0; i < size; ++i)
1238                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
1239         /* TOS. */
1240         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
1241         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
1242         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
1243         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
1244         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
1245         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
1246         /* Label. */
1247         if (inner) {
1248                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
1249                          vtc_m);
1250                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
1251                          vtc_v);
1252         } else {
1253                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
1254                          vtc_m);
1255                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
1256                          vtc_v);
1257         }
1258         /* Protocol. */
1259         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
1260                  ipv6_m->hdr.proto);
1261         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1262                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
1263 }
1264
1265 /**
1266  * Add TCP item to matcher and to the value.
1267  *
1268  * @param[in, out] matcher
1269  *   Flow matcher.
1270  * @param[in, out] key
1271  *   Flow matcher value.
1272  * @param[in] item
1273  *   Flow pattern to translate.
1274  * @param[in] inner
1275  *   Item is inner pattern.
1276  */
1277 static void
1278 flow_dv_translate_item_tcp(void *matcher, void *key,
1279                            const struct rte_flow_item *item,
1280                            int inner)
1281 {
1282         const struct rte_flow_item_tcp *tcp_m = item->mask;
1283         const struct rte_flow_item_tcp *tcp_v = item->spec;
1284         void *headers_m;
1285         void *headers_v;
1286
1287         if (inner) {
1288                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1289                                          inner_headers);
1290                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1291         } else {
1292                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1293                                          outer_headers);
1294                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1295         }
1296         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1297         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
1298         if (!tcp_v)
1299                 return;
1300         if (!tcp_m)
1301                 tcp_m = &rte_flow_item_tcp_mask;
1302         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
1303                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
1304         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
1305                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
1306         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
1307                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
1308         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
1309                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
1310 }
1311
1312 /**
1313  * Add UDP item to matcher and to the value.
1314  *
1315  * @param[in, out] matcher
1316  *   Flow matcher.
1317  * @param[in, out] key
1318  *   Flow matcher value.
1319  * @param[in] item
1320  *   Flow pattern to translate.
1321  * @param[in] inner
1322  *   Item is inner pattern.
1323  */
1324 static void
1325 flow_dv_translate_item_udp(void *matcher, void *key,
1326                            const struct rte_flow_item *item,
1327                            int inner)
1328 {
1329         const struct rte_flow_item_udp *udp_m = item->mask;
1330         const struct rte_flow_item_udp *udp_v = item->spec;
1331         void *headers_m;
1332         void *headers_v;
1333
1334         if (inner) {
1335                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1336                                          inner_headers);
1337                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1338         } else {
1339                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1340                                          outer_headers);
1341                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1342         }
1343         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1344         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
1345         if (!udp_v)
1346                 return;
1347         if (!udp_m)
1348                 udp_m = &rte_flow_item_udp_mask;
1349         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
1350                  rte_be_to_cpu_16(udp_m->hdr.src_port));
1351         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
1352                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
1353         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
1354                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
1355         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
1356                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
1357 }
1358
1359 /**
1360  * Add GRE item to matcher and to the value.
1361  *
1362  * @param[in, out] matcher
1363  *   Flow matcher.
1364  * @param[in, out] key
1365  *   Flow matcher value.
1366  * @param[in] item
1367  *   Flow pattern to translate.
1368  * @param[in] inner
1369  *   Item is inner pattern.
1370  */
1371 static void
1372 flow_dv_translate_item_gre(void *matcher, void *key,
1373                            const struct rte_flow_item *item,
1374                            int inner)
1375 {
1376         const struct rte_flow_item_gre *gre_m = item->mask;
1377         const struct rte_flow_item_gre *gre_v = item->spec;
1378         void *headers_m;
1379         void *headers_v;
1380         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1381         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1382
1383         if (inner) {
1384                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1385                                          inner_headers);
1386                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1387         } else {
1388                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1389                                          outer_headers);
1390                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1391         }
1392         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1393         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
1394         if (!gre_v)
1395                 return;
1396         if (!gre_m)
1397                 gre_m = &rte_flow_item_gre_mask;
1398         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
1399                  rte_be_to_cpu_16(gre_m->protocol));
1400         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
1401                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
1402 }
1403
1404 /**
1405  * Add NVGRE item to matcher and to the value.
1406  *
1407  * @param[in, out] matcher
1408  *   Flow matcher.
1409  * @param[in, out] key
1410  *   Flow matcher value.
1411  * @param[in] item
1412  *   Flow pattern to translate.
1413  * @param[in] inner
1414  *   Item is inner pattern.
1415  */
1416 static void
1417 flow_dv_translate_item_nvgre(void *matcher, void *key,
1418                              const struct rte_flow_item *item,
1419                              int inner)
1420 {
1421         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
1422         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
1423         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1424         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1425         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
1426         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
1427         char *gre_key_m;
1428         char *gre_key_v;
1429         int size;
1430         int i;
1431
1432         flow_dv_translate_item_gre(matcher, key, item, inner);
1433         if (!nvgre_v)
1434                 return;
1435         if (!nvgre_m)
1436                 nvgre_m = &rte_flow_item_nvgre_mask;
1437         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
1438         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
1439         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
1440         memcpy(gre_key_m, tni_flow_id_m, size);
1441         for (i = 0; i < size; ++i)
1442                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
1443 }
1444
1445 /**
1446  * Add VXLAN item to matcher and to the value.
1447  *
1448  * @param[in, out] matcher
1449  *   Flow matcher.
1450  * @param[in, out] key
1451  *   Flow matcher value.
1452  * @param[in] item
1453  *   Flow pattern to translate.
1454  * @param[in] inner
1455  *   Item is inner pattern.
1456  */
1457 static void
1458 flow_dv_translate_item_vxlan(void *matcher, void *key,
1459                              const struct rte_flow_item *item,
1460                              int inner)
1461 {
1462         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
1463         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
1464         void *headers_m;
1465         void *headers_v;
1466         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1467         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1468         char *vni_m;
1469         char *vni_v;
1470         uint16_t dport;
1471         int size;
1472         int i;
1473
1474         if (inner) {
1475                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1476                                          inner_headers);
1477                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1478         } else {
1479                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1480                                          outer_headers);
1481                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1482         }
1483         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
1484                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
1485         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
1486                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
1487                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
1488         }
1489         if (!vxlan_v)
1490                 return;
1491         if (!vxlan_m)
1492                 vxlan_m = &rte_flow_item_vxlan_mask;
1493         size = sizeof(vxlan_m->vni);
1494         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
1495         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
1496         memcpy(vni_m, vxlan_m->vni, size);
1497         for (i = 0; i < size; ++i)
1498                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
1499 }
1500
1501 /**
1502  * Add META item to matcher
1503  *
1504  * @param[in, out] matcher
1505  *   Flow matcher.
1506  * @param[in, out] key
1507  *   Flow matcher value.
1508  * @param[in] item
1509  *   Flow pattern to translate.
1510  * @param[in] inner
1511  *   Item is inner pattern.
1512  */
1513 static void
1514 flow_dv_translate_item_meta(void *matcher, void *key,
1515                             const struct rte_flow_item *item)
1516 {
1517         const struct rte_flow_item_meta *meta_m;
1518         const struct rte_flow_item_meta *meta_v;
1519         void *misc2_m =
1520                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
1521         void *misc2_v =
1522                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
1523
1524         meta_m = (const void *)item->mask;
1525         if (!meta_m)
1526                 meta_m = &rte_flow_item_meta_mask;
1527         meta_v = (const void *)item->spec;
1528         if (meta_v) {
1529                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
1530                          rte_be_to_cpu_32(meta_m->data));
1531                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
1532                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
1533         }
1534 }
1535
1536 /**
1537  * Update the matcher and the value based the selected item.
1538  *
1539  * @param[in, out] matcher
1540  *   Flow matcher.
1541  * @param[in, out] key
1542  *   Flow matcher value.
1543  * @param[in] item
1544  *   Flow pattern to translate.
1545  * @param[in, out] dev_flow
1546  *   Pointer to the mlx5_flow.
1547  * @param[in] inner
1548  *   Item is inner pattern.
1549  */
1550 static void
1551 flow_dv_create_item(void *matcher, void *key,
1552                     const struct rte_flow_item *item,
1553                     struct mlx5_flow *dev_flow,
1554                     int inner)
1555 {
1556         struct mlx5_flow_dv_matcher *tmatcher = matcher;
1557
1558         switch (item->type) {
1559         case RTE_FLOW_ITEM_TYPE_ETH:
1560                 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
1561                                            inner);
1562                 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
1563                 break;
1564         case RTE_FLOW_ITEM_TYPE_VLAN:
1565                 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
1566                                             inner);
1567                 break;
1568         case RTE_FLOW_ITEM_TYPE_IPV4:
1569                 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
1570                                             inner);
1571                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1572                 dev_flow->dv.hash_fields |=
1573                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1574                                                     MLX5_IPV4_LAYER_TYPES,
1575                                                     MLX5_IPV4_IBV_RX_HASH);
1576                 break;
1577         case RTE_FLOW_ITEM_TYPE_IPV6:
1578                 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
1579                                             inner);
1580                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1581                 dev_flow->dv.hash_fields |=
1582                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1583                                                     MLX5_IPV6_LAYER_TYPES,
1584                                                     MLX5_IPV6_IBV_RX_HASH);
1585                 break;
1586         case RTE_FLOW_ITEM_TYPE_TCP:
1587                 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
1588                                            inner);
1589                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1590                 dev_flow->dv.hash_fields |=
1591                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1592                                                     ETH_RSS_TCP,
1593                                                     (IBV_RX_HASH_SRC_PORT_TCP |
1594                                                      IBV_RX_HASH_DST_PORT_TCP));
1595                 break;
1596         case RTE_FLOW_ITEM_TYPE_UDP:
1597                 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
1598                                            inner);
1599                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1600                 dev_flow->verbs.hash_fields |=
1601                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1602                                                     ETH_RSS_UDP,
1603                                                     (IBV_RX_HASH_SRC_PORT_UDP |
1604                                                      IBV_RX_HASH_DST_PORT_UDP));
1605                 break;
1606         case RTE_FLOW_ITEM_TYPE_GRE:
1607                 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
1608                                            inner);
1609                 break;
1610         case RTE_FLOW_ITEM_TYPE_NVGRE:
1611                 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
1612                                              inner);
1613                 break;
1614         case RTE_FLOW_ITEM_TYPE_VXLAN:
1615         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1616                 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
1617                                              inner);
1618                 break;
1619         case RTE_FLOW_ITEM_TYPE_META:
1620                 flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
1621                 break;
1622         default:
1623                 break;
1624         }
1625 }
1626
1627 /**
1628  * Store the requested actions in an array.
1629  *
1630  * @param[in] dev
1631  *   Pointer to rte_eth_dev structure.
1632  * @param[in] action
1633  *   Flow action to translate.
1634  * @param[in, out] dev_flow
1635  *   Pointer to the mlx5_flow.
1636  * @param[in] attr
1637  *   Pointer to the flow attributes.
1638  * @param[out] error
1639  *   Pointer to the error structure.
1640  *
1641  * @return
1642  *   0 on success, a negative errno value otherwise and rte_errno is set.
1643  */
1644 static int
1645 flow_dv_create_action(struct rte_eth_dev *dev,
1646                       const struct rte_flow_action *action,
1647                       struct mlx5_flow *dev_flow,
1648                       const struct rte_flow_attr *attr,
1649                       struct rte_flow_error *error)
1650 {
1651         const struct rte_flow_action_queue *queue;
1652         const struct rte_flow_action_rss *rss;
1653         int actions_n = dev_flow->dv.actions_n;
1654         struct rte_flow *flow = dev_flow->flow;
1655         const struct rte_flow_action *action_ptr = action;
1656
1657         switch (action->type) {
1658         case RTE_FLOW_ACTION_TYPE_VOID:
1659                 break;
1660         case RTE_FLOW_ACTION_TYPE_FLAG:
1661                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1662                 dev_flow->dv.actions[actions_n].tag_value =
1663                         mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
1664                 actions_n++;
1665                 flow->actions |= MLX5_FLOW_ACTION_FLAG;
1666                 break;
1667         case RTE_FLOW_ACTION_TYPE_MARK:
1668                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1669                 dev_flow->dv.actions[actions_n].tag_value =
1670                         mlx5_flow_mark_set
1671                         (((const struct rte_flow_action_mark *)
1672                           (action->conf))->id);
1673                 flow->actions |= MLX5_FLOW_ACTION_MARK;
1674                 actions_n++;
1675                 break;
1676         case RTE_FLOW_ACTION_TYPE_DROP:
1677                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
1678                 flow->actions |= MLX5_FLOW_ACTION_DROP;
1679                 break;
1680         case RTE_FLOW_ACTION_TYPE_QUEUE:
1681                 queue = action->conf;
1682                 flow->rss.queue_num = 1;
1683                 (*flow->queue)[0] = queue->index;
1684                 flow->actions |= MLX5_FLOW_ACTION_QUEUE;
1685                 break;
1686         case RTE_FLOW_ACTION_TYPE_RSS:
1687                 rss = action->conf;
1688                 if (flow->queue)
1689                         memcpy((*flow->queue), rss->queue,
1690                                rss->queue_num * sizeof(uint16_t));
1691                 flow->rss.queue_num = rss->queue_num;
1692                 memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
1693                 flow->rss.types = rss->types;
1694                 flow->rss.level = rss->level;
1695                 /* Added to array only in apply since we need the QP */
1696                 flow->actions |= MLX5_FLOW_ACTION_RSS;
1697                 break;
1698         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1699         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
1700                 dev_flow->dv.actions[actions_n].type =
1701                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1702                 dev_flow->dv.actions[actions_n].action =
1703                                 flow_dv_create_action_l2_encap(dev, action,
1704                                                                error);
1705                 if (!(dev_flow->dv.actions[actions_n].action))
1706                         return -rte_errno;
1707                 dev_flow->dv.encap_decap_verbs_action =
1708                         dev_flow->dv.actions[actions_n].action;
1709                 flow->actions |= action->type ==
1710                                  RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
1711                                  MLX5_FLOW_ACTION_VXLAN_ENCAP :
1712                                  MLX5_FLOW_ACTION_NVGRE_ENCAP;
1713                 actions_n++;
1714                 break;
1715         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1716         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
1717                 dev_flow->dv.actions[actions_n].type =
1718                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1719                 dev_flow->dv.actions[actions_n].action =
1720                                 flow_dv_create_action_l2_decap(dev, error);
1721                 if (!(dev_flow->dv.actions[actions_n].action))
1722                         return -rte_errno;
1723                 dev_flow->dv.encap_decap_verbs_action =
1724                         dev_flow->dv.actions[actions_n].action;
1725                 flow->actions |= action->type ==
1726                                  RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
1727                                  MLX5_FLOW_ACTION_VXLAN_DECAP :
1728                                  MLX5_FLOW_ACTION_NVGRE_DECAP;
1729                 actions_n++;
1730                 break;
1731         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
1732                 /* Handle encap action with preceding decap */
1733                 if (flow->actions & MLX5_FLOW_ACTION_RAW_DECAP) {
1734                         dev_flow->dv.actions[actions_n].type =
1735                                 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1736                         dev_flow->dv.actions[actions_n].action =
1737                                         flow_dv_create_action_raw_encap
1738                                                                 (dev, action,
1739                                                                  attr, error);
1740                         if (!(dev_flow->dv.actions[actions_n].action))
1741                                 return -rte_errno;
1742                         dev_flow->dv.encap_decap_verbs_action =
1743                                 dev_flow->dv.actions[actions_n].action;
1744                 } else {
1745                         /* Handle encap action without preceding decap */
1746                         dev_flow->dv.actions[actions_n].type =
1747                                 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1748                         dev_flow->dv.actions[actions_n].action =
1749                                         flow_dv_create_action_l2_encap
1750                                                         (dev, action, error);
1751                         if (!(dev_flow->dv.actions[actions_n].action))
1752                                 return -rte_errno;
1753                         dev_flow->dv.encap_decap_verbs_action =
1754                                 dev_flow->dv.actions[actions_n].action;
1755                 }
1756                 flow->actions |= MLX5_FLOW_ACTION_RAW_ENCAP;
1757                 actions_n++;
1758                 break;
1759         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
1760                 /* Check if this decap action is followed by encap. */
1761                 for (; action_ptr->type != RTE_FLOW_ACTION_TYPE_END &&
1762                        action_ptr->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1763                        action_ptr++) {
1764                 }
1765                 /* Handle decap action only if it isn't followed by encap */
1766                 if (action_ptr->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1767                         dev_flow->dv.actions[actions_n].type =
1768                                 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1769                         dev_flow->dv.actions[actions_n].action =
1770                                         flow_dv_create_action_l2_decap(dev,
1771                                                                        error);
1772                         if (!(dev_flow->dv.actions[actions_n].action))
1773                                 return -rte_errno;
1774                         dev_flow->dv.encap_decap_verbs_action =
1775                                 dev_flow->dv.actions[actions_n].action;
1776                         actions_n++;
1777                 }
1778                 /* If decap is followed by encap, handle it at encap case. */
1779                 flow->actions |= MLX5_FLOW_ACTION_RAW_DECAP;
1780                 break;
1781         default:
1782                 break;
1783         }
1784         dev_flow->dv.actions_n = actions_n;
1785         return 0;
1786 }
1787
1788 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
1789
1790 #define HEADER_IS_ZERO(match_criteria, headers)                              \
1791         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
1792                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1793
1794 /**
1795  * Calculate flow matcher enable bitmap.
1796  *
1797  * @param match_criteria
1798  *   Pointer to flow matcher criteria.
1799  *
1800  * @return
1801  *   Bitmap of enabled fields.
1802  */
1803 static uint8_t
1804 flow_dv_matcher_enable(uint32_t *match_criteria)
1805 {
1806         uint8_t match_criteria_enable;
1807
1808         match_criteria_enable =
1809                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1810                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
1811         match_criteria_enable |=
1812                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1813                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
1814         match_criteria_enable |=
1815                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1816                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
1817         match_criteria_enable |=
1818                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
1819                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
1820
1821         return match_criteria_enable;
1822 }
1823
1824 /**
1825  * Register the flow matcher.
1826  *
1827  * @param dev[in, out]
1828  *   Pointer to rte_eth_dev structure.
1829  * @param[in, out] matcher
1830  *   Pointer to flow matcher.
1831  * @parm[in, out] dev_flow
1832  *   Pointer to the dev_flow.
1833  * @param[out] error
1834  *   pointer to error structure.
1835  *
1836  * @return
1837  *   0 on success otherwise -errno and errno is set.
1838  */
1839 static int
1840 flow_dv_matcher_register(struct rte_eth_dev *dev,
1841                          struct mlx5_flow_dv_matcher *matcher,
1842                          struct mlx5_flow *dev_flow,
1843                          struct rte_flow_error *error)
1844 {
1845         struct priv *priv = dev->data->dev_private;
1846         struct mlx5_flow_dv_matcher *cache_matcher;
1847         struct mlx5dv_flow_matcher_attr dv_attr = {
1848                 .type = IBV_FLOW_ATTR_NORMAL,
1849                 .match_mask = (void *)&matcher->mask,
1850         };
1851
1852         /* Lookup from cache. */
1853         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1854                 if (matcher->crc == cache_matcher->crc &&
1855                     matcher->priority == cache_matcher->priority &&
1856                     matcher->egress == cache_matcher->egress &&
1857                     !memcmp((const void *)matcher->mask.buf,
1858                             (const void *)cache_matcher->mask.buf,
1859                             cache_matcher->mask.size)) {
1860                         DRV_LOG(DEBUG,
1861                                 "priority %hd use %s matcher %p: refcnt %d++",
1862                                 cache_matcher->priority,
1863                                 cache_matcher->egress ? "tx" : "rx",
1864                                 (void *)cache_matcher,
1865                                 rte_atomic32_read(&cache_matcher->refcnt));
1866                         rte_atomic32_inc(&cache_matcher->refcnt);
1867                         dev_flow->dv.matcher = cache_matcher;
1868                         return 0;
1869                 }
1870         }
1871         /* Register new matcher. */
1872         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1873         if (!cache_matcher)
1874                 return rte_flow_error_set(error, ENOMEM,
1875                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1876                                           "cannot allocate matcher memory");
1877         *cache_matcher = *matcher;
1878         dv_attr.match_criteria_enable =
1879                 flow_dv_matcher_enable(cache_matcher->mask.buf);
1880         dv_attr.priority = matcher->priority;
1881         if (matcher->egress)
1882                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1883         cache_matcher->matcher_object =
1884                 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
1885         if (!cache_matcher->matcher_object) {
1886                 rte_free(cache_matcher);
1887                 return rte_flow_error_set(error, ENOMEM,
1888                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1889                                           NULL, "cannot create matcher");
1890         }
1891         rte_atomic32_inc(&cache_matcher->refcnt);
1892         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1893         dev_flow->dv.matcher = cache_matcher;
1894         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1895                 cache_matcher->priority,
1896                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1897                 rte_atomic32_read(&cache_matcher->refcnt));
1898         return 0;
1899 }
1900
1901 /**
1902  * Fill the flow with DV spec.
1903  *
1904  * @param[in] dev
1905  *   Pointer to rte_eth_dev structure.
1906  * @param[in, out] dev_flow
1907  *   Pointer to the sub flow.
1908  * @param[in] attr
1909  *   Pointer to the flow attributes.
1910  * @param[in] items
1911  *   Pointer to the list of items.
1912  * @param[in] actions
1913  *   Pointer to the list of actions.
1914  * @param[out] error
1915  *   Pointer to the error structure.
1916  *
1917  * @return
1918  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1919  */
1920 static int
1921 flow_dv_translate(struct rte_eth_dev *dev,
1922                   struct mlx5_flow *dev_flow,
1923                   const struct rte_flow_attr *attr,
1924                   const struct rte_flow_item items[],
1925                   const struct rte_flow_action actions[],
1926                   struct rte_flow_error *error)
1927 {
1928         struct priv *priv = dev->data->dev_private;
1929         uint64_t priority = attr->priority;
1930         struct mlx5_flow_dv_matcher matcher = {
1931                 .mask = {
1932                         .size = sizeof(matcher.mask.buf),
1933                 },
1934         };
1935         void *match_value = dev_flow->dv.value.buf;
1936         int tunnel = 0;
1937
1938         if (priority == MLX5_FLOW_PRIO_RSVD)
1939                 priority = priv->config.flow_prio - 1;
1940         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1941                 tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
1942                 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1943                                     tunnel);
1944         }
1945         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1946                                      matcher.mask.size);
1947         if (priority == MLX5_FLOW_PRIO_RSVD)
1948                 priority = priv->config.flow_prio - 1;
1949         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1950                                                      matcher.priority);
1951         matcher.egress = attr->egress;
1952         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1953                 return -rte_errno;
1954         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1955                 if (flow_dv_create_action(dev, actions, dev_flow, attr, error))
1956                         return -rte_errno;
1957         return 0;
1958 }
1959
1960 /**
1961  * Apply the flow to the NIC.
1962  *
1963  * @param[in] dev
1964  *   Pointer to the Ethernet device structure.
1965  * @param[in, out] flow
1966  *   Pointer to flow structure.
1967  * @param[out] error
1968  *   Pointer to error structure.
1969  *
1970  * @return
1971  *   0 on success, a negative errno value otherwise and rte_errno is set.
1972  */
1973 static int
1974 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1975               struct rte_flow_error *error)
1976 {
1977         struct mlx5_flow_dv *dv;
1978         struct mlx5_flow *dev_flow;
1979         int n;
1980         int err;
1981
1982         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1983                 dv = &dev_flow->dv;
1984                 n = dv->actions_n;
1985                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1986                         dv->hrxq = mlx5_hrxq_drop_new(dev);
1987                         if (!dv->hrxq) {
1988                                 rte_flow_error_set
1989                                         (error, errno,
1990                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1991                                          "cannot get drop hash queue");
1992                                 goto error;
1993                         }
1994                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1995                         dv->actions[n].qp = dv->hrxq->qp;
1996                         n++;
1997                 } else if (flow->actions &
1998                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
1999                         struct mlx5_hrxq *hrxq;
2000                         hrxq = mlx5_hrxq_get(dev, flow->key,
2001                                              MLX5_RSS_HASH_KEY_LEN,
2002                                              dv->hash_fields,
2003                                              (*flow->queue),
2004                                              flow->rss.queue_num);
2005                         if (!hrxq)
2006                                 hrxq = mlx5_hrxq_new
2007                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
2008                                          dv->hash_fields, (*flow->queue),
2009                                          flow->rss.queue_num,
2010                                          !!(dev_flow->layers &
2011                                             MLX5_FLOW_LAYER_TUNNEL));
2012                         if (!hrxq) {
2013                                 rte_flow_error_set
2014                                         (error, rte_errno,
2015                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2016                                          "cannot get hash queue");
2017                                 goto error;
2018                         }
2019                         dv->hrxq = hrxq;
2020                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
2021                         dv->actions[n].qp = hrxq->qp;
2022                         n++;
2023                 }
2024                 dv->flow =
2025                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
2026                                                   (void *)&dv->value, n,
2027                                                   dv->actions);
2028                 if (!dv->flow) {
2029                         rte_flow_error_set(error, errno,
2030                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2031                                            NULL,
2032                                            "hardware refuses to create flow");
2033                         goto error;
2034                 }
2035         }
2036         return 0;
2037 error:
2038         err = rte_errno; /* Save rte_errno before cleanup. */
2039         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
2040                 struct mlx5_flow_dv *dv = &dev_flow->dv;
2041                 if (dv->hrxq) {
2042                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
2043                                 mlx5_hrxq_drop_release(dev);
2044                         else
2045                                 mlx5_hrxq_release(dev, dv->hrxq);
2046                         dv->hrxq = NULL;
2047                 }
2048         }
2049         rte_errno = err; /* Restore rte_errno. */
2050         return -rte_errno;
2051 }
2052
2053 /**
2054  * Release the flow matcher.
2055  *
2056  * @param dev
2057  *   Pointer to Ethernet device.
2058  * @param flow
2059  *   Pointer to mlx5_flow.
2060  *
2061  * @return
2062  *   1 while a reference on it exists, 0 when freed.
2063  */
2064 static int
2065 flow_dv_matcher_release(struct rte_eth_dev *dev,
2066                         struct mlx5_flow *flow)
2067 {
2068         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
2069
2070         assert(matcher->matcher_object);
2071         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
2072                 dev->data->port_id, (void *)matcher,
2073                 rte_atomic32_read(&matcher->refcnt));
2074         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
2075                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
2076                            (matcher->matcher_object));
2077                 LIST_REMOVE(matcher, next);
2078                 rte_free(matcher);
2079                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
2080                         dev->data->port_id, (void *)matcher);
2081                 return 0;
2082         }
2083         return 1;
2084 }
2085
2086 /**
2087  * Remove the flow from the NIC but keeps it in memory.
2088  *
2089  * @param[in] dev
2090  *   Pointer to Ethernet device.
2091  * @param[in, out] flow
2092  *   Pointer to flow structure.
2093  */
2094 static void
2095 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
2096 {
2097         struct mlx5_flow_dv *dv;
2098         struct mlx5_flow *dev_flow;
2099
2100         if (!flow)
2101                 return;
2102         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
2103                 dv = &dev_flow->dv;
2104                 if (dv->flow) {
2105                         claim_zero(mlx5_glue->destroy_flow(dv->flow));
2106                         dv->flow = NULL;
2107                 }
2108                 if (dv->hrxq) {
2109                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
2110                                 mlx5_hrxq_drop_release(dev);
2111                         else
2112                                 mlx5_hrxq_release(dev, dv->hrxq);
2113                         dv->hrxq = NULL;
2114                 }
2115         }
2116         if (flow->counter)
2117                 flow->counter = NULL;
2118 }
2119
2120 /**
2121  * Remove the flow from the NIC and the memory.
2122  *
2123  * @param[in] dev
2124  *   Pointer to the Ethernet device structure.
2125  * @param[in, out] flow
2126  *   Pointer to flow structure.
2127  */
2128 static void
2129 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
2130 {
2131         struct mlx5_flow *dev_flow;
2132
2133         if (!flow)
2134                 return;
2135         flow_dv_remove(dev, flow);
2136         while (!LIST_EMPTY(&flow->dev_flows)) {
2137                 dev_flow = LIST_FIRST(&flow->dev_flows);
2138                 LIST_REMOVE(dev_flow, next);
2139                 if (dev_flow->dv.matcher)
2140                         flow_dv_matcher_release(dev, dev_flow);
2141                 if (dev_flow->dv.encap_decap_verbs_action) {
2142                         claim_zero(mlx5_glue->destroy_flow_action
2143                                 (dev_flow->dv.encap_decap_verbs_action));
2144                         dev_flow->dv.encap_decap_verbs_action = NULL;
2145                 }
2146                 rte_free(dev_flow);
2147         }
2148 }
2149
2150 /**
2151  * Query a flow.
2152  *
2153  * @see rte_flow_query()
2154  * @see rte_flow_ops
2155  */
2156 static int
2157 flow_dv_query(struct rte_eth_dev *dev __rte_unused,
2158               struct rte_flow *flow __rte_unused,
2159               const struct rte_flow_action *actions __rte_unused,
2160               void *data __rte_unused,
2161               struct rte_flow_error *error __rte_unused)
2162 {
2163         rte_errno = ENOTSUP;
2164         return -rte_errno;
2165 }
2166
2167
2168 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
2169         .validate = flow_dv_validate,
2170         .prepare = flow_dv_prepare,
2171         .translate = flow_dv_translate,
2172         .apply = flow_dv_apply,
2173         .remove = flow_dv_remove,
2174         .destroy = flow_dv_destroy,
2175         .query = flow_dv_query,
2176 };
2177
2178 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */