drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include "mlx5.h"
  30 #include "mlx5_defs.h"
  31 #include "mlx5_prm.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_flow.h"
  34
  35 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  36
  37 #define MLX5_ENCAP_MAX_LEN 132
  38
  39 /**
  40  * Validate META item.
  41  *
  42  * @param[in] dev
  43  *   Pointer to the rte_eth_dev structure.
  44  * @param[in] item
  45  *   Item specification.
  46  * @param[in] attr
  47  *   Attributes of flow that includes this item.
  48  * @param[out] error
  49  *   Pointer to error structure.
  50  *
  51  * @return
  52  *   0 on success, a negative errno value otherwise and rte_errno is set.
  53  */
  54 static int
  55 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
  56                            const struct rte_flow_item *item,
  57                            const struct rte_flow_attr *attr,
  58                            struct rte_flow_error *error)
  59 {
  60         const struct rte_flow_item_meta *spec = item->spec;
  61         const struct rte_flow_item_meta *mask = item->mask;
  62         const struct rte_flow_item_meta nic_mask = {
  63                 .data = RTE_BE32(UINT32_MAX)
  64         };
  65         int ret;
  66         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
  67
  68         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
  69                 return rte_flow_error_set(error, EPERM,
  70                                           RTE_FLOW_ERROR_TYPE_ITEM,
  71                                           NULL,
  72                                           "match on metadata offload "
  73                                           "configuration is off for this port");
  74         if (!spec)
  75                 return rte_flow_error_set(error, EINVAL,
  76                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  77                                           item->spec,
  78                                           "data cannot be empty");
  79         if (!spec->data)
  80                 return rte_flow_error_set(error, EINVAL,
  81                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  82                                           NULL,
  83                                           "data cannot be zero");
  84         if (!mask)
  85                 mask = &rte_flow_item_meta_mask;
  86         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
  87                                         (const uint8_t *)&nic_mask,
  88                                         sizeof(struct rte_flow_item_meta),
  89                                         error);
  90         if (ret < 0)
  91                 return ret;
  92         if (attr->ingress)
  93                 return rte_flow_error_set(error, ENOTSUP,
  94                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
  95                                           NULL,
  96                                           "pattern not supported for ingress");
  97         return 0;
  98 }
  99
 100 /**
 101  * Validate the L2 encap action.
 102  *
 103  * @param[in] action_flags
 104  *   Holds the actions detected until now.
 105  * @param[in] action
 106  *   Pointer to the encap action.
 107  * @param[in] attr
 108  *   Pointer to flow attributes
 109  * @param[out] error
 110  *   Pointer to error structure.
 111  *
 112  * @return
 113  *   0 on success, a negative errno value otherwise and rte_errno is set.
 114  */
 115 static int
 116 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 117                                  const struct rte_flow_action *action,
 118                                  const struct rte_flow_attr *attr,
 119                                  struct rte_flow_error *error)
 120 {
 121         if (!(action->conf))
 122                 return rte_flow_error_set(error, EINVAL,
 123                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 124                                           "configuration cannot be null");
 125         if (action_flags & MLX5_FLOW_ACTION_DROP)
 126                 return rte_flow_error_set(error, EINVAL,
 127                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 128                                           "can't drop and encap in same flow");
 129         if (action_flags & (MLX5_FLOW_ACTION_VXLAN_ENCAP |
 130                             MLX5_FLOW_ACTION_VXLAN_DECAP))
 131                 return rte_flow_error_set(error, EINVAL,
 132                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 133                                           "can only have a single encap or"
 134                                           " decap action in a flow");
 135         if (attr->ingress)
 136                 return rte_flow_error_set(error, ENOTSUP,
 137                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 138                                           NULL,
 139                                           "encap action not supported for "
 140                                           "ingress");
 141         return 0;
 142 }
 143
 144 /**
 145  * Validate the L2 decap action.
 146  *
 147  * @param[in] action_flags
 148  *   Holds the actions detected until now.
 149  * @param[in] attr
 150  *   Pointer to flow attributes
 151  * @param[out] error
 152  *   Pointer to error structure.
 153  *
 154  * @return
 155  *   0 on success, a negative errno value otherwise and rte_errno is set.
 156  */
 157 static int
 158 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 159                                  const struct rte_flow_attr *attr,
 160                                  struct rte_flow_error *error)
 161 {
 162         if (action_flags & MLX5_FLOW_ACTION_DROP)
 163                 return rte_flow_error_set(error, EINVAL,
 164                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 165                                           "can't drop and decap in same flow");
 166         if (action_flags & (MLX5_FLOW_ACTION_VXLAN_ENCAP |
 167                             MLX5_FLOW_ACTION_VXLAN_DECAP))
 168                 return rte_flow_error_set(error, EINVAL,
 169                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 170                                           "can only have a single encap or"
 171                                           " decap action in a flow");
 172         if (attr->egress)
 173                 return rte_flow_error_set(error, ENOTSUP,
 174                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 175                                           NULL,
 176                                           "decap action not supported for "
 177                                           "egress");
 178         return 0;
 179 }
 180
 181 /**
 182  * Get the size of specific rte_flow_item_type
 183  *
 184  * @param[in] item_type
 185  *   Tested rte_flow_item_type.
 186  *
 187  * @return
 188  *   sizeof struct item_type, 0 if void or irrelevant.
 189  */
 190 static size_t
 191 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
 192 {
 193         size_t retval;
 194
 195         switch (item_type) {
 196         case RTE_FLOW_ITEM_TYPE_ETH:
 197                 retval = sizeof(struct rte_flow_item_eth);
 198                 break;
 199         case RTE_FLOW_ITEM_TYPE_VLAN:
 200                 retval = sizeof(struct rte_flow_item_vlan);
 201                 break;
 202         case RTE_FLOW_ITEM_TYPE_IPV4:
 203                 retval = sizeof(struct rte_flow_item_ipv4);
 204                 break;
 205         case RTE_FLOW_ITEM_TYPE_IPV6:
 206                 retval = sizeof(struct rte_flow_item_ipv6);
 207                 break;
 208         case RTE_FLOW_ITEM_TYPE_UDP:
 209                 retval = sizeof(struct rte_flow_item_udp);
 210                 break;
 211         case RTE_FLOW_ITEM_TYPE_TCP:
 212                 retval = sizeof(struct rte_flow_item_tcp);
 213                 break;
 214         case RTE_FLOW_ITEM_TYPE_VXLAN:
 215                 retval = sizeof(struct rte_flow_item_vxlan);
 216                 break;
 217         case RTE_FLOW_ITEM_TYPE_GRE:
 218                 retval = sizeof(struct rte_flow_item_gre);
 219                 break;
 220         case RTE_FLOW_ITEM_TYPE_NVGRE:
 221                 retval = sizeof(struct rte_flow_item_nvgre);
 222                 break;
 223         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 224                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
 225                 break;
 226         case RTE_FLOW_ITEM_TYPE_MPLS:
 227                 retval = sizeof(struct rte_flow_item_mpls);
 228                 break;
 229         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
 230         default:
 231                 retval = 0;
 232                 break;
 233         }
 234         return retval;
 235 }
 236
 237 #define MLX5_ENCAP_IPV4_VERSION         0x40
 238 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
 239 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
 240 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
 241 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
 242 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
 243 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
 244
 245 /**
 246  * Convert the encap action data from list of rte_flow_item to raw buffer
 247  *
 248  * @param[in] items
 249  *   Pointer to rte_flow_item objects list.
 250  * @param[out] buf
 251  *   Pointer to the output buffer.
 252  * @param[out] size
 253  *   Pointer to the output buffer size.
 254  * @param[out] error
 255  *   Pointer to the error structure.
 256  *
 257  * @return
 258  *   0 on success, a negative errno value otherwise and rte_errno is set.
 259  */
 260 static int
 261 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 262                            size_t *size, struct rte_flow_error *error)
 263 {
 264         struct ether_hdr *eth = NULL;
 265         struct vlan_hdr *vlan = NULL;
 266         struct ipv4_hdr *ipv4 = NULL;
 267         struct ipv6_hdr *ipv6 = NULL;
 268         struct udp_hdr *udp = NULL;
 269         struct vxlan_hdr *vxlan = NULL;
 270         struct vxlan_gpe_hdr *vxlan_gpe = NULL;
 271         size_t len;
 272         size_t temp_size = 0;
 273
 274         if (!items)
 275                 return rte_flow_error_set(error, EINVAL,
 276                                           RTE_FLOW_ERROR_TYPE_ACTION,
 277                                           NULL, "invalid empty data");
 278         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 279                 len = flow_dv_get_item_len(items->type);
 280                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
 281                         return rte_flow_error_set(error, EINVAL,
 282                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 283                                                   (void *)items->type,
 284                                                   "items total size is too big"
 285                                                   " for encap action");
 286                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
 287                 switch (items->type) {
 288                 case RTE_FLOW_ITEM_TYPE_ETH:
 289                         eth = (struct ether_hdr *)&buf[temp_size];
 290                         break;
 291                 case RTE_FLOW_ITEM_TYPE_VLAN:
 292                         vlan = (struct vlan_hdr *)&buf[temp_size];
 293                         if (!eth)
 294                                 return rte_flow_error_set(error, EINVAL,
 295                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 296                                                 (void *)items->type,
 297                                                 "eth header not found");
 298                         if (!eth->ether_type)
 299                                 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
 300                         break;
 301                 case RTE_FLOW_ITEM_TYPE_IPV4:
 302                         ipv4 = (struct ipv4_hdr *)&buf[temp_size];
 303                         if (!vlan && !eth)
 304                                 return rte_flow_error_set(error, EINVAL,
 305                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 306                                                 (void *)items->type,
 307                                                 "neither eth nor vlan"
 308                                                 " header found");
 309                         if (vlan && !vlan->eth_proto)
 310                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
 311                         else if (eth && !eth->ether_type)
 312                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
 313                         if (!ipv4->version_ihl)
 314                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
 315                                                     MLX5_ENCAP_IPV4_IHL_MIN;
 316                         if (!ipv4->time_to_live)
 317                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
 318                         break;
 319                 case RTE_FLOW_ITEM_TYPE_IPV6:
 320                         ipv6 = (struct ipv6_hdr *)&buf[temp_size];
 321                         if (!vlan && !eth)
 322                                 return rte_flow_error_set(error, EINVAL,
 323                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 324                                                 (void *)items->type,
 325                                                 "neither eth nor vlan"
 326                                                 " header found");
 327                         if (vlan && !vlan->eth_proto)
 328                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
 329                         else if (eth && !eth->ether_type)
 330                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
 331                         if (!ipv6->vtc_flow)
 332                                 ipv6->vtc_flow =
 333                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
 334                         if (!ipv6->hop_limits)
 335                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
 336                         break;
 337                 case RTE_FLOW_ITEM_TYPE_UDP:
 338                         udp = (struct udp_hdr *)&buf[temp_size];
 339                         if (!ipv4 && !ipv6)
 340                                 return rte_flow_error_set(error, EINVAL,
 341                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 342                                                 (void *)items->type,
 343                                                 "ip header not found");
 344                         if (ipv4 && !ipv4->next_proto_id)
 345                                 ipv4->next_proto_id = IPPROTO_UDP;
 346                         else if (ipv6 && !ipv6->proto)
 347                                 ipv6->proto = IPPROTO_UDP;
 348                         break;
 349                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 350                         vxlan = (struct vxlan_hdr *)&buf[temp_size];
 351                         if (!udp)
 352                                 return rte_flow_error_set(error, EINVAL,
 353                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 354                                                 (void *)items->type,
 355                                                 "udp header not found");
 356                         if (!udp->dst_port)
 357                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
 358                         if (!vxlan->vx_flags)
 359                                 vxlan->vx_flags =
 360                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
 361                         break;
 362                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 363                         vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
 364                         if (!udp)
 365                                 return rte_flow_error_set(error, EINVAL,
 366                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 367                                                 (void *)items->type,
 368                                                 "udp header not found");
 369                         if (!vxlan_gpe->proto)
 370                                 return rte_flow_error_set(error, EINVAL,
 371                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 372                                                 (void *)items->type,
 373                                                 "next protocol not found");
 374                         if (!udp->dst_port)
 375                                 udp->dst_port =
 376                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
 377                         if (!vxlan_gpe->vx_flags)
 378                                 vxlan_gpe->vx_flags =
 379                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
 380                         break;
 381                 case RTE_FLOW_ITEM_TYPE_VOID:
 382                         break;
 383                 default:
 384                         return rte_flow_error_set(error, EINVAL,
 385                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 386                                                   (void *)items->type,
 387                                                   "unsupported item type");
 388                         break;
 389                 }
 390                 temp_size += len;
 391         }
 392         *size = temp_size;
 393         return 0;
 394 }
 395
 396 /**
 397  * Convert L2 encap action to DV specification.
 398  *
 399  * @param[in] dev
 400  *   Pointer to rte_eth_dev structure.
 401  * @param[in] action
 402  *   Pointer to action structure.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 408  */
 409 static struct ibv_flow_action *
 410 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
 411                                const struct rte_flow_action *action,
 412                                struct rte_flow_error *error)
 413 {
 414         struct ibv_flow_action *verbs_action = NULL;
 415         const struct rte_flow_item *encap_data;
 416         struct priv *priv = dev->data->dev_private;
 417         uint8_t buf[MLX5_ENCAP_MAX_LEN];
 418         size_t size = 0;
 419         int convert_result = 0;
 420
 421         encap_data = ((const struct rte_flow_action_vxlan_encap *)
 422                                                 action->conf)->definition;
 423         convert_result = flow_dv_convert_encap_data(encap_data, buf,
 424                                                     &size, error);
 425         if (convert_result)
 426                 return NULL;
 427         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 428                 (priv->ctx, size, buf,
 429                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
 430                  MLX5DV_FLOW_TABLE_TYPE_NIC_TX);
 431         if (!verbs_action)
 432                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 433                                    NULL, "cannot create L2 encap action");
 434         return verbs_action;
 435 }
 436
 437 /**
 438  * Convert L2 decap action to DV specification.
 439  *
 440  * @param[in] dev
 441  *   Pointer to rte_eth_dev structure.
 442  * @param[out] error
 443  *   Pointer to the error structure.
 444  *
 445  * @return
 446  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 447  */
 448 static struct ibv_flow_action *
 449 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
 450                                struct rte_flow_error *error)
 451 {
 452         struct ibv_flow_action *verbs_action = NULL;
 453         struct priv *priv = dev->data->dev_private;
 454
 455         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 456                 (priv->ctx, 0, NULL,
 457                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
 458                  MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
 459         if (!verbs_action)
 460                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 461                                    NULL, "cannot create L2 decap action");
 462         return verbs_action;
 463 }
 464
 465 /**
 466  * Verify the @p attributes will be correctly understood by the NIC and store
 467  * them in the @p flow if everything is correct.
 468  *
 469  * @param[in] dev
 470  *   Pointer to dev struct.
 471  * @param[in] attributes
 472  *   Pointer to flow attributes
 473  * @param[out] error
 474  *   Pointer to error structure.
 475  *
 476  * @return
 477  *   0 on success, a negative errno value otherwise and rte_errno is set.
 478  */
 479 static int
 480 flow_dv_validate_attributes(struct rte_eth_dev *dev,
 481                             const struct rte_flow_attr *attributes,
 482                             struct rte_flow_error *error)
 483 {
 484         struct priv *priv = dev->data->dev_private;
 485         uint32_t priority_max = priv->config.flow_prio - 1;
 486
 487         if (attributes->group)
 488                 return rte_flow_error_set(error, ENOTSUP,
 489                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 490                                           NULL,
 491                                           "groups is not supported");
 492         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
 493             attributes->priority >= priority_max)
 494                 return rte_flow_error_set(error, ENOTSUP,
 495                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 496                                           NULL,
 497                                           "priority out of range");
 498         if (attributes->transfer)
 499                 return rte_flow_error_set(error, ENOTSUP,
 500                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
 501                                           NULL,
 502                                           "transfer is not supported");
 503         if (!(attributes->egress ^ attributes->ingress))
 504                 return rte_flow_error_set(error, ENOTSUP,
 505                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 506                                           "must specify exactly one of "
 507                                           "ingress or egress");
 508         return 0;
 509 }
 510
 511 /**
 512  * Internal validation function. For validating both actions and items.
 513  *
 514  * @param[in] dev
 515  *   Pointer to the rte_eth_dev structure.
 516  * @param[in] attr
 517  *   Pointer to the flow attributes.
 518  * @param[in] items
 519  *   Pointer to the list of items.
 520  * @param[in] actions
 521  *   Pointer to the list of actions.
 522  * @param[out] error
 523  *   Pointer to the error structure.
 524  *
 525  * @return
 526  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 527  */
 528 static int
 529 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 530                  const struct rte_flow_item items[],
 531                  const struct rte_flow_action actions[],
 532                  struct rte_flow_error *error)
 533 {
 534         int ret;
 535         uint64_t action_flags = 0;
 536         uint64_t item_flags = 0;
 537         int tunnel = 0;
 538         uint8_t next_protocol = 0xff;
 539         int actions_n = 0;
 540
 541         if (items == NULL)
 542                 return -1;
 543         ret = flow_dv_validate_attributes(dev, attr, error);
 544         if (ret < 0)
 545                 return ret;
 546         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 547                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 548                 switch (items->type) {
 549                 case RTE_FLOW_ITEM_TYPE_VOID:
 550                         break;
 551                 case RTE_FLOW_ITEM_TYPE_ETH:
 552                         ret = mlx5_flow_validate_item_eth(items, item_flags,
 553                                                           error);
 554                         if (ret < 0)
 555                                 return ret;
 556                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 557                                                MLX5_FLOW_LAYER_OUTER_L2;
 558                         break;
 559                 case RTE_FLOW_ITEM_TYPE_VLAN:
 560                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
 561                                                            error);
 562                         if (ret < 0)
 563                                 return ret;
 564                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
 565                                                MLX5_FLOW_LAYER_OUTER_VLAN;
 566                         break;
 567                 case RTE_FLOW_ITEM_TYPE_IPV4:
 568                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
 569                                                            error);
 570                         if (ret < 0)
 571                                 return ret;
 572                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 573                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 574                         if (items->mask != NULL &&
 575                             ((const struct rte_flow_item_ipv4 *)
 576                              items->mask)->hdr.next_proto_id)
 577                                 next_protocol =
 578                                         ((const struct rte_flow_item_ipv4 *)
 579                                          (items->spec))->hdr.next_proto_id;
 580                         break;
 581                 case RTE_FLOW_ITEM_TYPE_IPV6:
 582                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
 583                                                            error);
 584                         if (ret < 0)
 585                                 return ret;
 586                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 587                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 588                         if (items->mask != NULL &&
 589                             ((const struct rte_flow_item_ipv6 *)
 590                              items->mask)->hdr.proto)
 591                                 next_protocol =
 592                                         ((const struct rte_flow_item_ipv6 *)
 593                                          items->spec)->hdr.proto;
 594                         break;
 595                 case RTE_FLOW_ITEM_TYPE_TCP:
 596                         ret = mlx5_flow_validate_item_tcp
 597                                                 (items, item_flags,
 598                                                  next_protocol,
 599                                                  &rte_flow_item_tcp_mask,
 600                                                  error);
 601                         if (ret < 0)
 602                                 return ret;
 603                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 604                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
 605                         break;
 606                 case RTE_FLOW_ITEM_TYPE_UDP:
 607                         ret = mlx5_flow_validate_item_udp(items, item_flags,
 608                                                           next_protocol,
 609                                                           error);
 610                         if (ret < 0)
 611                                 return ret;
 612                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 613                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
 614                         break;
 615                 case RTE_FLOW_ITEM_TYPE_GRE:
 616                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 617                         ret = mlx5_flow_validate_item_gre(items, item_flags,
 618                                                           next_protocol, error);
 619                         if (ret < 0)
 620                                 return ret;
 621                         item_flags |= MLX5_FLOW_LAYER_GRE;
 622                         break;
 623                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 624                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
 625                                                             error);
 626                         if (ret < 0)
 627                                 return ret;
 628                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
 629                         break;
 630                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 631                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
 632                                                                 item_flags, dev,
 633                                                                 error);
 634                         if (ret < 0)
 635                                 return ret;
 636                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 637                         break;
 638                 case RTE_FLOW_ITEM_TYPE_META:
 639                         ret = flow_dv_validate_item_meta(dev, items, attr,
 640                                                          error);
 641                         if (ret < 0)
 642                                 return ret;
 643                         item_flags |= MLX5_FLOW_ITEM_METADATA;
 644                         break;
 645                 default:
 646                         return rte_flow_error_set(error, ENOTSUP,
 647                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 648                                                   NULL, "item not supported");
 649                 }
 650         }
 651         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 652                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
 653                         return rte_flow_error_set(error, ENOTSUP,
 654                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 655                                                   actions, "too many actions");
 656                 switch (actions->type) {
 657                 case RTE_FLOW_ACTION_TYPE_VOID:
 658                         break;
 659                 case RTE_FLOW_ACTION_TYPE_FLAG:
 660                         ret = mlx5_flow_validate_action_flag(action_flags,
 661                                                              attr, error);
 662                         if (ret < 0)
 663                                 return ret;
 664                         action_flags |= MLX5_FLOW_ACTION_FLAG;
 665                         ++actions_n;
 666                         break;
 667                 case RTE_FLOW_ACTION_TYPE_MARK:
 668                         ret = mlx5_flow_validate_action_mark(actions,
 669                                                              action_flags,
 670                                                              attr, error);
 671                         if (ret < 0)
 672                                 return ret;
 673                         action_flags |= MLX5_FLOW_ACTION_MARK;
 674                         ++actions_n;
 675                         break;
 676                 case RTE_FLOW_ACTION_TYPE_DROP:
 677                         ret = mlx5_flow_validate_action_drop(action_flags,
 678                                                              attr, error);
 679                         if (ret < 0)
 680                                 return ret;
 681                         action_flags |= MLX5_FLOW_ACTION_DROP;
 682                         ++actions_n;
 683                         break;
 684                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 685                         ret = mlx5_flow_validate_action_queue(actions,
 686                                                               action_flags, dev,
 687                                                               attr, error);
 688                         if (ret < 0)
 689                                 return ret;
 690                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
 691                         ++actions_n;
 692                         break;
 693                 case RTE_FLOW_ACTION_TYPE_RSS:
 694                         ret = mlx5_flow_validate_action_rss(actions,
 695                                                             action_flags, dev,
 696                                                             attr, error);
 697                         if (ret < 0)
 698                                 return ret;
 699                         action_flags |= MLX5_FLOW_ACTION_RSS;
 700                         ++actions_n;
 701                         break;
 702                 case RTE_FLOW_ACTION_TYPE_COUNT:
 703                         ret = mlx5_flow_validate_action_count(dev, attr, error);
 704                         if (ret < 0)
 705                                 return ret;
 706                         action_flags |= MLX5_FLOW_ACTION_COUNT;
 707                         ++actions_n;
 708                         break;
 709                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 710                         ret = flow_dv_validate_action_l2_encap(action_flags,
 711                                                                actions, attr,
 712                                                                error);
 713                         if (ret < 0)
 714                                 return ret;
 715                         action_flags |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
 716                         ++actions_n;
 717                         break;
 718                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
 719                         ret = flow_dv_validate_action_l2_decap(action_flags,
 720                                                                attr, error);
 721                         if (ret < 0)
 722                                 return ret;
 723                         action_flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
 724                         ++actions_n;
 725                         break;
 726                 default:
 727                         return rte_flow_error_set(error, ENOTSUP,
 728                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 729                                                   actions,
 730                                                   "action not supported");
 731                 }
 732         }
 733         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
 734                 return rte_flow_error_set(error, EINVAL,
 735                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
 736                                           "no fate action is found");
 737         return 0;
 738 }
 739
 740 /**
 741  * Internal preparation function. Allocates the DV flow size,
 742  * this size is constant.
 743  *
 744  * @param[in] attr
 745  *   Pointer to the flow attributes.
 746  * @param[in] items
 747  *   Pointer to the list of items.
 748  * @param[in] actions
 749  *   Pointer to the list of actions.
 750  * @param[out] item_flags
 751  *   Pointer to bit mask of all items detected.
 752  * @param[out] action_flags
 753  *   Pointer to bit mask of all actions detected.
 754  * @param[out] error
 755  *   Pointer to the error structure.
 756  *
 757  * @return
 758  *   Pointer to mlx5_flow object on success,
 759  *   otherwise NULL and rte_ernno is set.
 760  */
 761 static struct mlx5_flow *
 762 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
 763                 const struct rte_flow_item items[] __rte_unused,
 764                 const struct rte_flow_action actions[] __rte_unused,
 765                 uint64_t *item_flags __rte_unused,
 766                 uint64_t *action_flags __rte_unused,
 767                 struct rte_flow_error *error)
 768 {
 769         uint32_t size = sizeof(struct mlx5_flow);
 770         struct mlx5_flow *flow;
 771
 772         flow = rte_calloc(__func__, 1, size, 0);
 773         if (!flow) {
 774                 rte_flow_error_set(error, ENOMEM,
 775                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 776                                    "not enough memory to create flow");
 777                 return NULL;
 778         }
 779         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
 780         return flow;
 781 }
 782
 783 /**
 784  * Add Ethernet item to matcher and to the value.
 785  *
 786  * @param[in, out] matcher
 787  *   Flow matcher.
 788  * @param[in, out] key
 789  *   Flow matcher value.
 790  * @param[in] item
 791  *   Flow pattern to translate.
 792  * @param[in] inner
 793  *   Item is inner pattern.
 794  */
 795 static void
 796 flow_dv_translate_item_eth(void *matcher, void *key,
 797                            const struct rte_flow_item *item, int inner)
 798 {
 799         const struct rte_flow_item_eth *eth_m = item->mask;
 800         const struct rte_flow_item_eth *eth_v = item->spec;
 801         const struct rte_flow_item_eth nic_mask = {
 802                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 803                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 804                 .type = RTE_BE16(0xffff),
 805         };
 806         void *headers_m;
 807         void *headers_v;
 808         char *l24_v;
 809         unsigned int i;
 810
 811         if (!eth_v)
 812                 return;
 813         if (!eth_m)
 814                 eth_m = &nic_mask;
 815         if (inner) {
 816                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 817                                          inner_headers);
 818                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 819         } else {
 820                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 821                                          outer_headers);
 822                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 823         }
 824         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
 825                &eth_m->dst, sizeof(eth_m->dst));
 826         /* The value must be in the range of the mask. */
 827         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
 828         for (i = 0; i < sizeof(eth_m->dst); ++i)
 829                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
 830         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
 831                &eth_m->src, sizeof(eth_m->src));
 832         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
 833         /* The value must be in the range of the mask. */
 834         for (i = 0; i < sizeof(eth_m->dst); ++i)
 835                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
 836         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
 837                  rte_be_to_cpu_16(eth_m->type));
 838         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
 839         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
 840 }
 841
 842 /**
 843  * Add VLAN item to matcher and to the value.
 844  *
 845  * @param[in, out] matcher
 846  *   Flow matcher.
 847  * @param[in, out] key
 848  *   Flow matcher value.
 849  * @param[in] item
 850  *   Flow pattern to translate.
 851  * @param[in] inner
 852  *   Item is inner pattern.
 853  */
 854 static void
 855 flow_dv_translate_item_vlan(void *matcher, void *key,
 856                             const struct rte_flow_item *item,
 857                             int inner)
 858 {
 859         const struct rte_flow_item_vlan *vlan_m = item->mask;
 860         const struct rte_flow_item_vlan *vlan_v = item->spec;
 861         const struct rte_flow_item_vlan nic_mask = {
 862                 .tci = RTE_BE16(0x0fff),
 863                 .inner_type = RTE_BE16(0xffff),
 864         };
 865         void *headers_m;
 866         void *headers_v;
 867         uint16_t tci_m;
 868         uint16_t tci_v;
 869
 870         if (!vlan_v)
 871                 return;
 872         if (!vlan_m)
 873                 vlan_m = &nic_mask;
 874         if (inner) {
 875                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 876                                          inner_headers);
 877                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 878         } else {
 879                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 880                                          outer_headers);
 881                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 882         }
 883         tci_m = rte_be_to_cpu_16(vlan_m->tci);
 884         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
 885         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
 886         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
 887         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
 888         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
 889         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
 890         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
 891         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
 892         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
 893 }
 894
 895 /**
 896  * Add IPV4 item to matcher and to the value.
 897  *
 898  * @param[in, out] matcher
 899  *   Flow matcher.
 900  * @param[in, out] key
 901  *   Flow matcher value.
 902  * @param[in] item
 903  *   Flow pattern to translate.
 904  * @param[in] inner
 905  *   Item is inner pattern.
 906  */
 907 static void
 908 flow_dv_translate_item_ipv4(void *matcher, void *key,
 909                             const struct rte_flow_item *item,
 910                             int inner)
 911 {
 912         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
 913         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
 914         const struct rte_flow_item_ipv4 nic_mask = {
 915                 .hdr = {
 916                         .src_addr = RTE_BE32(0xffffffff),
 917                         .dst_addr = RTE_BE32(0xffffffff),
 918                         .type_of_service = 0xff,
 919                         .next_proto_id = 0xff,
 920                 },
 921         };
 922         void *headers_m;
 923         void *headers_v;
 924         char *l24_m;
 925         char *l24_v;
 926         uint8_t tos;
 927
 928         if (inner) {
 929                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 930                                          inner_headers);
 931                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 932         } else {
 933                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 934                                          outer_headers);
 935                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 936         }
 937         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 938         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
 939         if (!ipv4_v)
 940                 return;
 941         if (!ipv4_m)
 942                 ipv4_m = &nic_mask;
 943         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 944                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 945         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 946                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 947         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
 948         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
 949         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 950                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 951         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 952                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 953         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
 954         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
 955         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
 956         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
 957                  ipv4_m->hdr.type_of_service);
 958         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
 959         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
 960                  ipv4_m->hdr.type_of_service >> 2);
 961         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
 962         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 963                  ipv4_m->hdr.next_proto_id);
 964         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 965                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
 966 }
 967
 968 /**
 969  * Add IPV6 item to matcher and to the value.
 970  *
 971  * @param[in, out] matcher
 972  *   Flow matcher.
 973  * @param[in, out] key
 974  *   Flow matcher value.
 975  * @param[in] item
 976  *   Flow pattern to translate.
 977  * @param[in] inner
 978  *   Item is inner pattern.
 979  */
 980 static void
 981 flow_dv_translate_item_ipv6(void *matcher, void *key,
 982                             const struct rte_flow_item *item,
 983                             int inner)
 984 {
 985         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
 986         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
 987         const struct rte_flow_item_ipv6 nic_mask = {
 988                 .hdr = {
 989                         .src_addr =
 990                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
 991                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
 992                         .dst_addr =
 993                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
 994                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
 995                         .vtc_flow = RTE_BE32(0xffffffff),
 996                         .proto = 0xff,
 997                         .hop_limits = 0xff,
 998                 },
 999         };
1000         void *headers_m;
1001         void *headers_v;
1002         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1003         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1004         char *l24_m;
1005         char *l24_v;
1006         uint32_t vtc_m;
1007         uint32_t vtc_v;
1008         int i;
1009         int size;
1010
1011         if (inner) {
1012                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1013                                          inner_headers);
1014                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1015         } else {
1016                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1017                                          outer_headers);
1018                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1019         }
1020         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
1021         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
1022         if (!ipv6_v)
1023                 return;
1024         if (!ipv6_m)
1025                 ipv6_m = &nic_mask;
1026         size = sizeof(ipv6_m->hdr.dst_addr);
1027         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1028                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1029         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1030                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1031         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
1032         for (i = 0; i < size; ++i)
1033                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
1034         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1035                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1036         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1037                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1038         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
1039         for (i = 0; i < size; ++i)
1040                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
1041         /* TOS. */
1042         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
1043         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
1044         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
1045         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
1046         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
1047         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
1048         /* Label. */
1049         if (inner) {
1050                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
1051                          vtc_m);
1052                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
1053                          vtc_v);
1054         } else {
1055                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
1056                          vtc_m);
1057                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
1058                          vtc_v);
1059         }
1060         /* Protocol. */
1061         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
1062                  ipv6_m->hdr.proto);
1063         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1064                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
1065 }
1066
1067 /**
1068  * Add TCP item to matcher and to the value.
1069  *
1070  * @param[in, out] matcher
1071  *   Flow matcher.
1072  * @param[in, out] key
1073  *   Flow matcher value.
1074  * @param[in] item
1075  *   Flow pattern to translate.
1076  * @param[in] inner
1077  *   Item is inner pattern.
1078  */
1079 static void
1080 flow_dv_translate_item_tcp(void *matcher, void *key,
1081                            const struct rte_flow_item *item,
1082                            int inner)
1083 {
1084         const struct rte_flow_item_tcp *tcp_m = item->mask;
1085         const struct rte_flow_item_tcp *tcp_v = item->spec;
1086         void *headers_m;
1087         void *headers_v;
1088
1089         if (inner) {
1090                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1091                                          inner_headers);
1092                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1093         } else {
1094                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1095                                          outer_headers);
1096                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1097         }
1098         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1099         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
1100         if (!tcp_v)
1101                 return;
1102         if (!tcp_m)
1103                 tcp_m = &rte_flow_item_tcp_mask;
1104         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
1105                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
1106         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
1107                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
1108         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
1109                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
1110         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
1111                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
1112 }
1113
1114 /**
1115  * Add UDP item to matcher and to the value.
1116  *
1117  * @param[in, out] matcher
1118  *   Flow matcher.
1119  * @param[in, out] key
1120  *   Flow matcher value.
1121  * @param[in] item
1122  *   Flow pattern to translate.
1123  * @param[in] inner
1124  *   Item is inner pattern.
1125  */
1126 static void
1127 flow_dv_translate_item_udp(void *matcher, void *key,
1128                            const struct rte_flow_item *item,
1129                            int inner)
1130 {
1131         const struct rte_flow_item_udp *udp_m = item->mask;
1132         const struct rte_flow_item_udp *udp_v = item->spec;
1133         void *headers_m;
1134         void *headers_v;
1135
1136         if (inner) {
1137                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1138                                          inner_headers);
1139                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1140         } else {
1141                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1142                                          outer_headers);
1143                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1144         }
1145         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1146         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
1147         if (!udp_v)
1148                 return;
1149         if (!udp_m)
1150                 udp_m = &rte_flow_item_udp_mask;
1151         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
1152                  rte_be_to_cpu_16(udp_m->hdr.src_port));
1153         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
1154                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
1155         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
1156                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
1157         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
1158                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
1159 }
1160
1161 /**
1162  * Add GRE item to matcher and to the value.
1163  *
1164  * @param[in, out] matcher
1165  *   Flow matcher.
1166  * @param[in, out] key
1167  *   Flow matcher value.
1168  * @param[in] item
1169  *   Flow pattern to translate.
1170  * @param[in] inner
1171  *   Item is inner pattern.
1172  */
1173 static void
1174 flow_dv_translate_item_gre(void *matcher, void *key,
1175                            const struct rte_flow_item *item,
1176                            int inner)
1177 {
1178         const struct rte_flow_item_gre *gre_m = item->mask;
1179         const struct rte_flow_item_gre *gre_v = item->spec;
1180         void *headers_m;
1181         void *headers_v;
1182         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1183         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1184
1185         if (inner) {
1186                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1187                                          inner_headers);
1188                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1189         } else {
1190                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1191                                          outer_headers);
1192                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1193         }
1194         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1195         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
1196         if (!gre_v)
1197                 return;
1198         if (!gre_m)
1199                 gre_m = &rte_flow_item_gre_mask;
1200         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
1201                  rte_be_to_cpu_16(gre_m->protocol));
1202         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
1203                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
1204 }
1205
1206 /**
1207  * Add NVGRE item to matcher and to the value.
1208  *
1209  * @param[in, out] matcher
1210  *   Flow matcher.
1211  * @param[in, out] key
1212  *   Flow matcher value.
1213  * @param[in] item
1214  *   Flow pattern to translate.
1215  * @param[in] inner
1216  *   Item is inner pattern.
1217  */
1218 static void
1219 flow_dv_translate_item_nvgre(void *matcher, void *key,
1220                              const struct rte_flow_item *item,
1221                              int inner)
1222 {
1223         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
1224         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
1225         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1226         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1227         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
1228         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
1229         char *gre_key_m;
1230         char *gre_key_v;
1231         int size;
1232         int i;
1233
1234         flow_dv_translate_item_gre(matcher, key, item, inner);
1235         if (!nvgre_v)
1236                 return;
1237         if (!nvgre_m)
1238                 nvgre_m = &rte_flow_item_nvgre_mask;
1239         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
1240         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
1241         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
1242         memcpy(gre_key_m, tni_flow_id_m, size);
1243         for (i = 0; i < size; ++i)
1244                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
1245 }
1246
1247 /**
1248  * Add VXLAN item to matcher and to the value.
1249  *
1250  * @param[in, out] matcher
1251  *   Flow matcher.
1252  * @param[in, out] key
1253  *   Flow matcher value.
1254  * @param[in] item
1255  *   Flow pattern to translate.
1256  * @param[in] inner
1257  *   Item is inner pattern.
1258  */
1259 static void
1260 flow_dv_translate_item_vxlan(void *matcher, void *key,
1261                              const struct rte_flow_item *item,
1262                              int inner)
1263 {
1264         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
1265         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
1266         void *headers_m;
1267         void *headers_v;
1268         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1269         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1270         char *vni_m;
1271         char *vni_v;
1272         uint16_t dport;
1273         int size;
1274         int i;
1275
1276         if (inner) {
1277                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1278                                          inner_headers);
1279                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1280         } else {
1281                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1282                                          outer_headers);
1283                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1284         }
1285         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
1286                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
1287         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
1288                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
1289                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
1290         }
1291         if (!vxlan_v)
1292                 return;
1293         if (!vxlan_m)
1294                 vxlan_m = &rte_flow_item_vxlan_mask;
1295         size = sizeof(vxlan_m->vni);
1296         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
1297         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
1298         memcpy(vni_m, vxlan_m->vni, size);
1299         for (i = 0; i < size; ++i)
1300                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
1301 }
1302
1303 /**
1304  * Add META item to matcher
1305  *
1306  * @param[in, out] matcher
1307  *   Flow matcher.
1308  * @param[in, out] key
1309  *   Flow matcher value.
1310  * @param[in] item
1311  *   Flow pattern to translate.
1312  * @param[in] inner
1313  *   Item is inner pattern.
1314  */
1315 static void
1316 flow_dv_translate_item_meta(void *matcher, void *key,
1317                             const struct rte_flow_item *item)
1318 {
1319         const struct rte_flow_item_meta *meta_m;
1320         const struct rte_flow_item_meta *meta_v;
1321         void *misc2_m =
1322                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
1323         void *misc2_v =
1324                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
1325
1326         meta_m = (const void *)item->mask;
1327         if (!meta_m)
1328                 meta_m = &rte_flow_item_meta_mask;
1329         meta_v = (const void *)item->spec;
1330         if (meta_v) {
1331                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
1332                          rte_be_to_cpu_32(meta_m->data));
1333                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
1334                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
1335         }
1336 }
1337
1338 /**
1339  * Update the matcher and the value based the selected item.
1340  *
1341  * @param[in, out] matcher
1342  *   Flow matcher.
1343  * @param[in, out] key
1344  *   Flow matcher value.
1345  * @param[in] item
1346  *   Flow pattern to translate.
1347  * @param[in, out] dev_flow
1348  *   Pointer to the mlx5_flow.
1349  * @param[in] inner
1350  *   Item is inner pattern.
1351  */
1352 static void
1353 flow_dv_create_item(void *matcher, void *key,
1354                     const struct rte_flow_item *item,
1355                     struct mlx5_flow *dev_flow,
1356                     int inner)
1357 {
1358         struct mlx5_flow_dv_matcher *tmatcher = matcher;
1359
1360         switch (item->type) {
1361         case RTE_FLOW_ITEM_TYPE_ETH:
1362                 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
1363                                            inner);
1364                 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
1365                 break;
1366         case RTE_FLOW_ITEM_TYPE_VLAN:
1367                 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
1368                                             inner);
1369                 break;
1370         case RTE_FLOW_ITEM_TYPE_IPV4:
1371                 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
1372                                             inner);
1373                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1374                 dev_flow->dv.hash_fields |=
1375                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1376                                                     MLX5_IPV4_LAYER_TYPES,
1377                                                     MLX5_IPV4_IBV_RX_HASH);
1378                 break;
1379         case RTE_FLOW_ITEM_TYPE_IPV6:
1380                 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
1381                                             inner);
1382                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1383                 dev_flow->dv.hash_fields |=
1384                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1385                                                     MLX5_IPV6_LAYER_TYPES,
1386                                                     MLX5_IPV6_IBV_RX_HASH);
1387                 break;
1388         case RTE_FLOW_ITEM_TYPE_TCP:
1389                 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
1390                                            inner);
1391                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1392                 dev_flow->dv.hash_fields |=
1393                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1394                                                     ETH_RSS_TCP,
1395                                                     (IBV_RX_HASH_SRC_PORT_TCP |
1396                                                      IBV_RX_HASH_DST_PORT_TCP));
1397                 break;
1398         case RTE_FLOW_ITEM_TYPE_UDP:
1399                 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
1400                                            inner);
1401                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1402                 dev_flow->verbs.hash_fields |=
1403                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1404                                                     ETH_RSS_UDP,
1405                                                     (IBV_RX_HASH_SRC_PORT_UDP |
1406                                                      IBV_RX_HASH_DST_PORT_UDP));
1407                 break;
1408         case RTE_FLOW_ITEM_TYPE_GRE:
1409                 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
1410                                            inner);
1411                 break;
1412         case RTE_FLOW_ITEM_TYPE_NVGRE:
1413                 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
1414                                              inner);
1415                 break;
1416         case RTE_FLOW_ITEM_TYPE_VXLAN:
1417         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1418                 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
1419                                              inner);
1420                 break;
1421         case RTE_FLOW_ITEM_TYPE_META:
1422                 flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
1423                 break;
1424         default:
1425                 break;
1426         }
1427 }
1428
1429 /**
1430  * Store the requested actions in an array.
1431  *
1432  * @param[in] dev
1433  *   Pointer to rte_eth_dev structure.
1434  * @param[in] action
1435  *   Flow action to translate.
1436  * @param[in, out] dev_flow
1437  *   Pointer to the mlx5_flow.
1438  * @param[out] error
1439  *   Pointer to the error structure.
1440  *
1441  * @return
1442  *   0 on success, a negative errno value otherwise and rte_errno is set.
1443  */
1444 static int
1445 flow_dv_create_action(struct rte_eth_dev *dev,
1446                       const struct rte_flow_action *action,
1447                       struct mlx5_flow *dev_flow,
1448                       struct rte_flow_error *error)
1449 {
1450         const struct rte_flow_action_queue *queue;
1451         const struct rte_flow_action_rss *rss;
1452         int actions_n = dev_flow->dv.actions_n;
1453         struct rte_flow *flow = dev_flow->flow;
1454
1455         switch (action->type) {
1456         case RTE_FLOW_ACTION_TYPE_VOID:
1457                 break;
1458         case RTE_FLOW_ACTION_TYPE_FLAG:
1459                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1460                 dev_flow->dv.actions[actions_n].tag_value =
1461                         mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
1462                 actions_n++;
1463                 flow->actions |= MLX5_FLOW_ACTION_FLAG;
1464                 break;
1465         case RTE_FLOW_ACTION_TYPE_MARK:
1466                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1467                 dev_flow->dv.actions[actions_n].tag_value =
1468                         mlx5_flow_mark_set
1469                         (((const struct rte_flow_action_mark *)
1470                           (action->conf))->id);
1471                 flow->actions |= MLX5_FLOW_ACTION_MARK;
1472                 actions_n++;
1473                 break;
1474         case RTE_FLOW_ACTION_TYPE_DROP:
1475                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
1476                 flow->actions |= MLX5_FLOW_ACTION_DROP;
1477                 break;
1478         case RTE_FLOW_ACTION_TYPE_QUEUE:
1479                 queue = action->conf;
1480                 flow->rss.queue_num = 1;
1481                 (*flow->queue)[0] = queue->index;
1482                 flow->actions |= MLX5_FLOW_ACTION_QUEUE;
1483                 break;
1484         case RTE_FLOW_ACTION_TYPE_RSS:
1485                 rss = action->conf;
1486                 if (flow->queue)
1487                         memcpy((*flow->queue), rss->queue,
1488                                rss->queue_num * sizeof(uint16_t));
1489                 flow->rss.queue_num = rss->queue_num;
1490                 memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
1491                 flow->rss.types = rss->types;
1492                 flow->rss.level = rss->level;
1493                 /* Added to array only in apply since we need the QP */
1494                 flow->actions |= MLX5_FLOW_ACTION_RSS;
1495                 break;
1496         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1497                 dev_flow->dv.actions[actions_n].type =
1498                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1499                 dev_flow->dv.actions[actions_n].action =
1500                                 flow_dv_create_action_l2_encap(dev, action,
1501                                                                error);
1502                 if (!(dev_flow->dv.actions[actions_n].action))
1503                         return -rte_errno;
1504                 dev_flow->dv.encap_decap_verbs_action =
1505                         dev_flow->dv.actions[actions_n].action;
1506                 flow->actions |= MLX5_FLOW_ACTION_VXLAN_ENCAP;
1507                 actions_n++;
1508                 break;
1509         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1510                 dev_flow->dv.actions[actions_n].type =
1511                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1512                 dev_flow->dv.actions[actions_n].action =
1513                                 flow_dv_create_action_l2_decap(dev, error);
1514                 if (!(dev_flow->dv.actions[actions_n].action))
1515                         return -rte_errno;
1516                 dev_flow->dv.encap_decap_verbs_action =
1517                         dev_flow->dv.actions[actions_n].action;
1518                 flow->actions |= MLX5_FLOW_ACTION_VXLAN_DECAP;
1519                 actions_n++;
1520                 break;
1521         default:
1522                 break;
1523         }
1524         dev_flow->dv.actions_n = actions_n;
1525         return 0;
1526 }
1527
1528 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
1529
1530 #define HEADER_IS_ZERO(match_criteria, headers)                              \
1531         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
1532                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1533
1534 /**
1535  * Calculate flow matcher enable bitmap.
1536  *
1537  * @param match_criteria
1538  *   Pointer to flow matcher criteria.
1539  *
1540  * @return
1541  *   Bitmap of enabled fields.
1542  */
1543 static uint8_t
1544 flow_dv_matcher_enable(uint32_t *match_criteria)
1545 {
1546         uint8_t match_criteria_enable;
1547
1548         match_criteria_enable =
1549                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1550                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
1551         match_criteria_enable |=
1552                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1553                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
1554         match_criteria_enable |=
1555                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1556                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
1557         match_criteria_enable |=
1558                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
1559                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
1560
1561         return match_criteria_enable;
1562 }
1563
1564 /**
1565  * Register the flow matcher.
1566  *
1567  * @param dev[in, out]
1568  *   Pointer to rte_eth_dev structure.
1569  * @param[in, out] matcher
1570  *   Pointer to flow matcher.
1571  * @parm[in, out] dev_flow
1572  *   Pointer to the dev_flow.
1573  * @param[out] error
1574  *   pointer to error structure.
1575  *
1576  * @return
1577  *   0 on success otherwise -errno and errno is set.
1578  */
1579 static int
1580 flow_dv_matcher_register(struct rte_eth_dev *dev,
1581                          struct mlx5_flow_dv_matcher *matcher,
1582                          struct mlx5_flow *dev_flow,
1583                          struct rte_flow_error *error)
1584 {
1585         struct priv *priv = dev->data->dev_private;
1586         struct mlx5_flow_dv_matcher *cache_matcher;
1587         struct mlx5dv_flow_matcher_attr dv_attr = {
1588                 .type = IBV_FLOW_ATTR_NORMAL,
1589                 .match_mask = (void *)&matcher->mask,
1590         };
1591
1592         /* Lookup from cache. */
1593         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1594                 if (matcher->crc == cache_matcher->crc &&
1595                     matcher->priority == cache_matcher->priority &&
1596                     matcher->egress == cache_matcher->egress &&
1597                     !memcmp((const void *)matcher->mask.buf,
1598                             (const void *)cache_matcher->mask.buf,
1599                             cache_matcher->mask.size)) {
1600                         DRV_LOG(DEBUG,
1601                                 "priority %hd use %s matcher %p: refcnt %d++",
1602                                 cache_matcher->priority,
1603                                 cache_matcher->egress ? "tx" : "rx",
1604                                 (void *)cache_matcher,
1605                                 rte_atomic32_read(&cache_matcher->refcnt));
1606                         rte_atomic32_inc(&cache_matcher->refcnt);
1607                         dev_flow->dv.matcher = cache_matcher;
1608                         return 0;
1609                 }
1610         }
1611         /* Register new matcher. */
1612         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1613         if (!cache_matcher)
1614                 return rte_flow_error_set(error, ENOMEM,
1615                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1616                                           "cannot allocate matcher memory");
1617         *cache_matcher = *matcher;
1618         dv_attr.match_criteria_enable =
1619                 flow_dv_matcher_enable(cache_matcher->mask.buf);
1620         dv_attr.priority = matcher->priority;
1621         if (matcher->egress)
1622                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1623         cache_matcher->matcher_object =
1624                 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
1625         if (!cache_matcher->matcher_object) {
1626                 rte_free(cache_matcher);
1627                 return rte_flow_error_set(error, ENOMEM,
1628                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1629                                           NULL, "cannot create matcher");
1630         }
1631         rte_atomic32_inc(&cache_matcher->refcnt);
1632         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1633         dev_flow->dv.matcher = cache_matcher;
1634         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1635                 cache_matcher->priority,
1636                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1637                 rte_atomic32_read(&cache_matcher->refcnt));
1638         return 0;
1639 }
1640
1641 /**
1642  * Fill the flow with DV spec.
1643  *
1644  * @param[in] dev
1645  *   Pointer to rte_eth_dev structure.
1646  * @param[in, out] dev_flow
1647  *   Pointer to the sub flow.
1648  * @param[in] attr
1649  *   Pointer to the flow attributes.
1650  * @param[in] items
1651  *   Pointer to the list of items.
1652  * @param[in] actions
1653  *   Pointer to the list of actions.
1654  * @param[out] error
1655  *   Pointer to the error structure.
1656  *
1657  * @return
1658  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1659  */
1660 static int
1661 flow_dv_translate(struct rte_eth_dev *dev,
1662                   struct mlx5_flow *dev_flow,
1663                   const struct rte_flow_attr *attr,
1664                   const struct rte_flow_item items[],
1665                   const struct rte_flow_action actions[] __rte_unused,
1666                   struct rte_flow_error *error)
1667 {
1668         struct priv *priv = dev->data->dev_private;
1669         uint64_t priority = attr->priority;
1670         struct mlx5_flow_dv_matcher matcher = {
1671                 .mask = {
1672                         .size = sizeof(matcher.mask.buf),
1673                 },
1674         };
1675         void *match_value = dev_flow->dv.value.buf;
1676         int tunnel = 0;
1677
1678         if (priority == MLX5_FLOW_PRIO_RSVD)
1679                 priority = priv->config.flow_prio - 1;
1680         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1681                 tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
1682                 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1683                                     tunnel);
1684         }
1685         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1686                                      matcher.mask.size);
1687         if (priority == MLX5_FLOW_PRIO_RSVD)
1688                 priority = priv->config.flow_prio - 1;
1689         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1690                                                      matcher.priority);
1691         matcher.egress = attr->egress;
1692         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1693                 return -rte_errno;
1694         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1695                 if (flow_dv_create_action(dev, actions, dev_flow, error))
1696                         return -rte_errno;
1697         return 0;
1698 }
1699
1700 /**
1701  * Apply the flow to the NIC.
1702  *
1703  * @param[in] dev
1704  *   Pointer to the Ethernet device structure.
1705  * @param[in, out] flow
1706  *   Pointer to flow structure.
1707  * @param[out] error
1708  *   Pointer to error structure.
1709  *
1710  * @return
1711  *   0 on success, a negative errno value otherwise and rte_errno is set.
1712  */
1713 static int
1714 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1715               struct rte_flow_error *error)
1716 {
1717         struct mlx5_flow_dv *dv;
1718         struct mlx5_flow *dev_flow;
1719         int n;
1720         int err;
1721
1722         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1723                 dv = &dev_flow->dv;
1724                 n = dv->actions_n;
1725                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1726                         dv->hrxq = mlx5_hrxq_drop_new(dev);
1727                         if (!dv->hrxq) {
1728                                 rte_flow_error_set
1729                                         (error, errno,
1730                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1731                                          "cannot get drop hash queue");
1732                                 goto error;
1733                         }
1734                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1735                         dv->actions[n].qp = dv->hrxq->qp;
1736                         n++;
1737                 } else if (flow->actions &
1738                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
1739                         struct mlx5_hrxq *hrxq;
1740                         hrxq = mlx5_hrxq_get(dev, flow->key,
1741                                              MLX5_RSS_HASH_KEY_LEN,
1742                                              dv->hash_fields,
1743                                              (*flow->queue),
1744                                              flow->rss.queue_num);
1745                         if (!hrxq)
1746                                 hrxq = mlx5_hrxq_new
1747                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
1748                                          dv->hash_fields, (*flow->queue),
1749                                          flow->rss.queue_num,
1750                                          !!(dev_flow->layers &
1751                                             MLX5_FLOW_LAYER_TUNNEL));
1752                         if (!hrxq) {
1753                                 rte_flow_error_set
1754                                         (error, rte_errno,
1755                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1756                                          "cannot get hash queue");
1757                                 goto error;
1758                         }
1759                         dv->hrxq = hrxq;
1760                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1761                         dv->actions[n].qp = hrxq->qp;
1762                         n++;
1763                 }
1764                 dv->flow =
1765                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
1766                                                   (void *)&dv->value, n,
1767                                                   dv->actions);
1768                 if (!dv->flow) {
1769                         rte_flow_error_set(error, errno,
1770                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1771                                            NULL,
1772                                            "hardware refuses to create flow");
1773                         goto error;
1774                 }
1775         }
1776         return 0;
1777 error:
1778         err = rte_errno; /* Save rte_errno before cleanup. */
1779         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1780                 struct mlx5_flow_dv *dv = &dev_flow->dv;
1781                 if (dv->hrxq) {
1782                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1783                                 mlx5_hrxq_drop_release(dev);
1784                         else
1785                                 mlx5_hrxq_release(dev, dv->hrxq);
1786                         dv->hrxq = NULL;
1787                 }
1788         }
1789         rte_errno = err; /* Restore rte_errno. */
1790         return -rte_errno;
1791 }
1792
1793 /**
1794  * Release the flow matcher.
1795  *
1796  * @param dev
1797  *   Pointer to Ethernet device.
1798  * @param flow
1799  *   Pointer to mlx5_flow.
1800  *
1801  * @return
1802  *   1 while a reference on it exists, 0 when freed.
1803  */
1804 static int
1805 flow_dv_matcher_release(struct rte_eth_dev *dev,
1806                         struct mlx5_flow *flow)
1807 {
1808         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
1809
1810         assert(matcher->matcher_object);
1811         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
1812                 dev->data->port_id, (void *)matcher,
1813                 rte_atomic32_read(&matcher->refcnt));
1814         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
1815                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
1816                            (matcher->matcher_object));
1817                 LIST_REMOVE(matcher, next);
1818                 rte_free(matcher);
1819                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
1820                         dev->data->port_id, (void *)matcher);
1821                 return 0;
1822         }
1823         return 1;
1824 }
1825
1826 /**
1827  * Remove the flow from the NIC but keeps it in memory.
1828  *
1829  * @param[in] dev
1830  *   Pointer to Ethernet device.
1831  * @param[in, out] flow
1832  *   Pointer to flow structure.
1833  */
1834 static void
1835 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1836 {
1837         struct mlx5_flow_dv *dv;
1838         struct mlx5_flow *dev_flow;
1839
1840         if (!flow)
1841                 return;
1842         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1843                 dv = &dev_flow->dv;
1844                 if (dv->flow) {
1845                         claim_zero(mlx5_glue->destroy_flow(dv->flow));
1846                         dv->flow = NULL;
1847                 }
1848                 if (dv->hrxq) {
1849                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1850                                 mlx5_hrxq_drop_release(dev);
1851                         else
1852                                 mlx5_hrxq_release(dev, dv->hrxq);
1853                         dv->hrxq = NULL;
1854                 }
1855         }
1856         if (flow->counter)
1857                 flow->counter = NULL;
1858 }
1859
1860 /**
1861  * Remove the flow from the NIC and the memory.
1862  *
1863  * @param[in] dev
1864  *   Pointer to the Ethernet device structure.
1865  * @param[in, out] flow
1866  *   Pointer to flow structure.
1867  */
1868 static void
1869 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1870 {
1871         struct mlx5_flow *dev_flow;
1872
1873         if (!flow)
1874                 return;
1875         flow_dv_remove(dev, flow);
1876         while (!LIST_EMPTY(&flow->dev_flows)) {
1877                 dev_flow = LIST_FIRST(&flow->dev_flows);
1878                 LIST_REMOVE(dev_flow, next);
1879                 if (dev_flow->dv.matcher)
1880                         flow_dv_matcher_release(dev, dev_flow);
1881                 if (dev_flow->dv.encap_decap_verbs_action) {
1882                         claim_zero(mlx5_glue->destroy_flow_action
1883                                 (dev_flow->dv.encap_decap_verbs_action));
1884                         dev_flow->dv.encap_decap_verbs_action = NULL;
1885                 }
1886                 rte_free(dev_flow);
1887         }
1888 }
1889
1890 /**
1891  * Query a flow.
1892  *
1893  * @see rte_flow_query()
1894  * @see rte_flow_ops
1895  */
1896 static int
1897 flow_dv_query(struct rte_eth_dev *dev __rte_unused,
1898               struct rte_flow *flow __rte_unused,
1899               const struct rte_flow_action *actions __rte_unused,
1900               void *data __rte_unused,
1901               struct rte_flow_error *error __rte_unused)
1902 {
1903         rte_errno = ENOTSUP;
1904         return -rte_errno;
1905 }
1906
1907
1908 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
1909         .validate = flow_dv_validate,
1910         .prepare = flow_dv_prepare,
1911         .translate = flow_dv_translate,
1912         .apply = flow_dv_apply,
1913         .remove = flow_dv_remove,
1914         .destroy = flow_dv_destroy,
1915         .query = flow_dv_query,
1916 };
1917
1918 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */