drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  37
  38 #define MLX5_ENCAP_MAX_LEN 132
  39
  40 /**
  41  * Validate META item.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the rte_eth_dev structure.
  45  * @param[in] item
  46  *   Item specification.
  47  * @param[in] attr
  48  *   Attributes of flow that includes this item.
  49  * @param[out] error
  50  *   Pointer to error structure.
  51  *
  52  * @return
  53  *   0 on success, a negative errno value otherwise and rte_errno is set.
  54  */
  55 static int
  56 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
  57                            const struct rte_flow_item *item,
  58                            const struct rte_flow_attr *attr,
  59                            struct rte_flow_error *error)
  60 {
  61         const struct rte_flow_item_meta *spec = item->spec;
  62         const struct rte_flow_item_meta *mask = item->mask;
  63         const struct rte_flow_item_meta nic_mask = {
  64                 .data = RTE_BE32(UINT32_MAX)
  65         };
  66         int ret;
  67         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
  68
  69         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
  70                 return rte_flow_error_set(error, EPERM,
  71                                           RTE_FLOW_ERROR_TYPE_ITEM,
  72                                           NULL,
  73                                           "match on metadata offload "
  74                                           "configuration is off for this port");
  75         if (!spec)
  76                 return rte_flow_error_set(error, EINVAL,
  77                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  78                                           item->spec,
  79                                           "data cannot be empty");
  80         if (!spec->data)
  81                 return rte_flow_error_set(error, EINVAL,
  82                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  83                                           NULL,
  84                                           "data cannot be zero");
  85         if (!mask)
  86                 mask = &rte_flow_item_meta_mask;
  87         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
  88                                         (const uint8_t *)&nic_mask,
  89                                         sizeof(struct rte_flow_item_meta),
  90                                         error);
  91         if (ret < 0)
  92                 return ret;
  93         if (attr->ingress)
  94                 return rte_flow_error_set(error, ENOTSUP,
  95                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
  96                                           NULL,
  97                                           "pattern not supported for ingress");
  98         return 0;
  99 }
 100
 101 /**
 102  * Validate the L2 encap action.
 103  *
 104  * @param[in] action_flags
 105  *   Holds the actions detected until now.
 106  * @param[in] action
 107  *   Pointer to the encap action.
 108  * @param[in] attr
 109  *   Pointer to flow attributes
 110  * @param[out] error
 111  *   Pointer to error structure.
 112  *
 113  * @return
 114  *   0 on success, a negative errno value otherwise and rte_errno is set.
 115  */
 116 static int
 117 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 118                                  const struct rte_flow_action *action,
 119                                  const struct rte_flow_attr *attr,
 120                                  struct rte_flow_error *error)
 121 {
 122         if (!(action->conf))
 123                 return rte_flow_error_set(error, EINVAL,
 124                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 125                                           "configuration cannot be null");
 126         if (action_flags & MLX5_FLOW_ACTION_DROP)
 127                 return rte_flow_error_set(error, EINVAL,
 128                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 129                                           "can't drop and encap in same flow");
 130         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 131                 return rte_flow_error_set(error, EINVAL,
 132                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 133                                           "can only have a single encap or"
 134                                           " decap action in a flow");
 135         if (attr->ingress)
 136                 return rte_flow_error_set(error, ENOTSUP,
 137                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 138                                           NULL,
 139                                           "encap action not supported for "
 140                                           "ingress");
 141         return 0;
 142 }
 143
 144 /**
 145  * Validate the L2 decap action.
 146  *
 147  * @param[in] action_flags
 148  *   Holds the actions detected until now.
 149  * @param[in] attr
 150  *   Pointer to flow attributes
 151  * @param[out] error
 152  *   Pointer to error structure.
 153  *
 154  * @return
 155  *   0 on success, a negative errno value otherwise and rte_errno is set.
 156  */
 157 static int
 158 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 159                                  const struct rte_flow_attr *attr,
 160                                  struct rte_flow_error *error)
 161 {
 162         if (action_flags & MLX5_FLOW_ACTION_DROP)
 163                 return rte_flow_error_set(error, EINVAL,
 164                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 165                                           "can't drop and decap in same flow");
 166         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 167                 return rte_flow_error_set(error, EINVAL,
 168                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 169                                           "can only have a single encap or"
 170                                           " decap action in a flow");
 171         if (attr->egress)
 172                 return rte_flow_error_set(error, ENOTSUP,
 173                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 174                                           NULL,
 175                                           "decap action not supported for "
 176                                           "egress");
 177         return 0;
 178 }
 179
 180 /**
 181  * Get the size of specific rte_flow_item_type
 182  *
 183  * @param[in] item_type
 184  *   Tested rte_flow_item_type.
 185  *
 186  * @return
 187  *   sizeof struct item_type, 0 if void or irrelevant.
 188  */
 189 static size_t
 190 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
 191 {
 192         size_t retval;
 193
 194         switch (item_type) {
 195         case RTE_FLOW_ITEM_TYPE_ETH:
 196                 retval = sizeof(struct rte_flow_item_eth);
 197                 break;
 198         case RTE_FLOW_ITEM_TYPE_VLAN:
 199                 retval = sizeof(struct rte_flow_item_vlan);
 200                 break;
 201         case RTE_FLOW_ITEM_TYPE_IPV4:
 202                 retval = sizeof(struct rte_flow_item_ipv4);
 203                 break;
 204         case RTE_FLOW_ITEM_TYPE_IPV6:
 205                 retval = sizeof(struct rte_flow_item_ipv6);
 206                 break;
 207         case RTE_FLOW_ITEM_TYPE_UDP:
 208                 retval = sizeof(struct rte_flow_item_udp);
 209                 break;
 210         case RTE_FLOW_ITEM_TYPE_TCP:
 211                 retval = sizeof(struct rte_flow_item_tcp);
 212                 break;
 213         case RTE_FLOW_ITEM_TYPE_VXLAN:
 214                 retval = sizeof(struct rte_flow_item_vxlan);
 215                 break;
 216         case RTE_FLOW_ITEM_TYPE_GRE:
 217                 retval = sizeof(struct rte_flow_item_gre);
 218                 break;
 219         case RTE_FLOW_ITEM_TYPE_NVGRE:
 220                 retval = sizeof(struct rte_flow_item_nvgre);
 221                 break;
 222         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 223                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
 224                 break;
 225         case RTE_FLOW_ITEM_TYPE_MPLS:
 226                 retval = sizeof(struct rte_flow_item_mpls);
 227                 break;
 228         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
 229         default:
 230                 retval = 0;
 231                 break;
 232         }
 233         return retval;
 234 }
 235
 236 #define MLX5_ENCAP_IPV4_VERSION         0x40
 237 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
 238 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
 239 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
 240 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
 241 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
 242 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
 243
 244 /**
 245  * Convert the encap action data from list of rte_flow_item to raw buffer
 246  *
 247  * @param[in] items
 248  *   Pointer to rte_flow_item objects list.
 249  * @param[out] buf
 250  *   Pointer to the output buffer.
 251  * @param[out] size
 252  *   Pointer to the output buffer size.
 253  * @param[out] error
 254  *   Pointer to the error structure.
 255  *
 256  * @return
 257  *   0 on success, a negative errno value otherwise and rte_errno is set.
 258  */
 259 static int
 260 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 261                            size_t *size, struct rte_flow_error *error)
 262 {
 263         struct ether_hdr *eth = NULL;
 264         struct vlan_hdr *vlan = NULL;
 265         struct ipv4_hdr *ipv4 = NULL;
 266         struct ipv6_hdr *ipv6 = NULL;
 267         struct udp_hdr *udp = NULL;
 268         struct vxlan_hdr *vxlan = NULL;
 269         struct vxlan_gpe_hdr *vxlan_gpe = NULL;
 270         struct gre_hdr *gre = NULL;
 271         size_t len;
 272         size_t temp_size = 0;
 273
 274         if (!items)
 275                 return rte_flow_error_set(error, EINVAL,
 276                                           RTE_FLOW_ERROR_TYPE_ACTION,
 277                                           NULL, "invalid empty data");
 278         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 279                 len = flow_dv_get_item_len(items->type);
 280                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
 281                         return rte_flow_error_set(error, EINVAL,
 282                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 283                                                   (void *)items->type,
 284                                                   "items total size is too big"
 285                                                   " for encap action");
 286                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
 287                 switch (items->type) {
 288                 case RTE_FLOW_ITEM_TYPE_ETH:
 289                         eth = (struct ether_hdr *)&buf[temp_size];
 290                         break;
 291                 case RTE_FLOW_ITEM_TYPE_VLAN:
 292                         vlan = (struct vlan_hdr *)&buf[temp_size];
 293                         if (!eth)
 294                                 return rte_flow_error_set(error, EINVAL,
 295                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 296                                                 (void *)items->type,
 297                                                 "eth header not found");
 298                         if (!eth->ether_type)
 299                                 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
 300                         break;
 301                 case RTE_FLOW_ITEM_TYPE_IPV4:
 302                         ipv4 = (struct ipv4_hdr *)&buf[temp_size];
 303                         if (!vlan && !eth)
 304                                 return rte_flow_error_set(error, EINVAL,
 305                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 306                                                 (void *)items->type,
 307                                                 "neither eth nor vlan"
 308                                                 " header found");
 309                         if (vlan && !vlan->eth_proto)
 310                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
 311                         else if (eth && !eth->ether_type)
 312                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
 313                         if (!ipv4->version_ihl)
 314                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
 315                                                     MLX5_ENCAP_IPV4_IHL_MIN;
 316                         if (!ipv4->time_to_live)
 317                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
 318                         break;
 319                 case RTE_FLOW_ITEM_TYPE_IPV6:
 320                         ipv6 = (struct ipv6_hdr *)&buf[temp_size];
 321                         if (!vlan && !eth)
 322                                 return rte_flow_error_set(error, EINVAL,
 323                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 324                                                 (void *)items->type,
 325                                                 "neither eth nor vlan"
 326                                                 " header found");
 327                         if (vlan && !vlan->eth_proto)
 328                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
 329                         else if (eth && !eth->ether_type)
 330                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
 331                         if (!ipv6->vtc_flow)
 332                                 ipv6->vtc_flow =
 333                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
 334                         if (!ipv6->hop_limits)
 335                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
 336                         break;
 337                 case RTE_FLOW_ITEM_TYPE_UDP:
 338                         udp = (struct udp_hdr *)&buf[temp_size];
 339                         if (!ipv4 && !ipv6)
 340                                 return rte_flow_error_set(error, EINVAL,
 341                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 342                                                 (void *)items->type,
 343                                                 "ip header not found");
 344                         if (ipv4 && !ipv4->next_proto_id)
 345                                 ipv4->next_proto_id = IPPROTO_UDP;
 346                         else if (ipv6 && !ipv6->proto)
 347                                 ipv6->proto = IPPROTO_UDP;
 348                         break;
 349                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 350                         vxlan = (struct vxlan_hdr *)&buf[temp_size];
 351                         if (!udp)
 352                                 return rte_flow_error_set(error, EINVAL,
 353                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 354                                                 (void *)items->type,
 355                                                 "udp header not found");
 356                         if (!udp->dst_port)
 357                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
 358                         if (!vxlan->vx_flags)
 359                                 vxlan->vx_flags =
 360                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
 361                         break;
 362                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 363                         vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
 364                         if (!udp)
 365                                 return rte_flow_error_set(error, EINVAL,
 366                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 367                                                 (void *)items->type,
 368                                                 "udp header not found");
 369                         if (!vxlan_gpe->proto)
 370                                 return rte_flow_error_set(error, EINVAL,
 371                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 372                                                 (void *)items->type,
 373                                                 "next protocol not found");
 374                         if (!udp->dst_port)
 375                                 udp->dst_port =
 376                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
 377                         if (!vxlan_gpe->vx_flags)
 378                                 vxlan_gpe->vx_flags =
 379                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
 380                         break;
 381                 case RTE_FLOW_ITEM_TYPE_GRE:
 382                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 383                         gre = (struct gre_hdr *)&buf[temp_size];
 384                         if (!gre->proto)
 385                                 return rte_flow_error_set(error, EINVAL,
 386                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 387                                                 (void *)items->type,
 388                                                 "next protocol not found");
 389                         if (!ipv4 && !ipv6)
 390                                 return rte_flow_error_set(error, EINVAL,
 391                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 392                                                 (void *)items->type,
 393                                                 "ip header not found");
 394                         if (ipv4 && !ipv4->next_proto_id)
 395                                 ipv4->next_proto_id = IPPROTO_GRE;
 396                         else if (ipv6 && !ipv6->proto)
 397                                 ipv6->proto = IPPROTO_GRE;
 398                         break;
 399                 case RTE_FLOW_ITEM_TYPE_VOID:
 400                         break;
 401                 default:
 402                         return rte_flow_error_set(error, EINVAL,
 403                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 404                                                   (void *)items->type,
 405                                                   "unsupported item type");
 406                         break;
 407                 }
 408                 temp_size += len;
 409         }
 410         *size = temp_size;
 411         return 0;
 412 }
 413
 414 /**
 415  * Convert L2 encap action to DV specification.
 416  *
 417  * @param[in] dev
 418  *   Pointer to rte_eth_dev structure.
 419  * @param[in] action
 420  *   Pointer to action structure.
 421  * @param[out] error
 422  *   Pointer to the error structure.
 423  *
 424  * @return
 425  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 426  */
 427 static struct ibv_flow_action *
 428 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
 429                                const struct rte_flow_action *action,
 430                                struct rte_flow_error *error)
 431 {
 432         struct ibv_flow_action *verbs_action = NULL;
 433         const struct rte_flow_item *encap_data;
 434         struct priv *priv = dev->data->dev_private;
 435         uint8_t buf[MLX5_ENCAP_MAX_LEN];
 436         size_t size = 0;
 437         int convert_result = 0;
 438
 439         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
 440                 encap_data = ((const struct rte_flow_action_vxlan_encap *)
 441                                                 action->conf)->definition;
 442         else
 443                 encap_data = ((const struct rte_flow_action_nvgre_encap *)
 444                                                 action->conf)->definition;
 445         convert_result = flow_dv_convert_encap_data(encap_data, buf,
 446                                                     &size, error);
 447         if (convert_result)
 448                 return NULL;
 449         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 450                 (priv->ctx, size, buf,
 451                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
 452                  MLX5DV_FLOW_TABLE_TYPE_NIC_TX);
 453         if (!verbs_action)
 454                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 455                                    NULL, "cannot create L2 encap action");
 456         return verbs_action;
 457 }
 458
 459 /**
 460  * Convert L2 decap action to DV specification.
 461  *
 462  * @param[in] dev
 463  *   Pointer to rte_eth_dev structure.
 464  * @param[out] error
 465  *   Pointer to the error structure.
 466  *
 467  * @return
 468  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 469  */
 470 static struct ibv_flow_action *
 471 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
 472                                struct rte_flow_error *error)
 473 {
 474         struct ibv_flow_action *verbs_action = NULL;
 475         struct priv *priv = dev->data->dev_private;
 476
 477         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 478                 (priv->ctx, 0, NULL,
 479                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
 480                  MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
 481         if (!verbs_action)
 482                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 483                                    NULL, "cannot create L2 decap action");
 484         return verbs_action;
 485 }
 486
 487 /**
 488  * Verify the @p attributes will be correctly understood by the NIC and store
 489  * them in the @p flow if everything is correct.
 490  *
 491  * @param[in] dev
 492  *   Pointer to dev struct.
 493  * @param[in] attributes
 494  *   Pointer to flow attributes
 495  * @param[out] error
 496  *   Pointer to error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_validate_attributes(struct rte_eth_dev *dev,
 503                             const struct rte_flow_attr *attributes,
 504                             struct rte_flow_error *error)
 505 {
 506         struct priv *priv = dev->data->dev_private;
 507         uint32_t priority_max = priv->config.flow_prio - 1;
 508
 509         if (attributes->group)
 510                 return rte_flow_error_set(error, ENOTSUP,
 511                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 512                                           NULL,
 513                                           "groups is not supported");
 514         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
 515             attributes->priority >= priority_max)
 516                 return rte_flow_error_set(error, ENOTSUP,
 517                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 518                                           NULL,
 519                                           "priority out of range");
 520         if (attributes->transfer)
 521                 return rte_flow_error_set(error, ENOTSUP,
 522                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
 523                                           NULL,
 524                                           "transfer is not supported");
 525         if (!(attributes->egress ^ attributes->ingress))
 526                 return rte_flow_error_set(error, ENOTSUP,
 527                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 528                                           "must specify exactly one of "
 529                                           "ingress or egress");
 530         return 0;
 531 }
 532
 533 /**
 534  * Internal validation function. For validating both actions and items.
 535  *
 536  * @param[in] dev
 537  *   Pointer to the rte_eth_dev structure.
 538  * @param[in] attr
 539  *   Pointer to the flow attributes.
 540  * @param[in] items
 541  *   Pointer to the list of items.
 542  * @param[in] actions
 543  *   Pointer to the list of actions.
 544  * @param[out] error
 545  *   Pointer to the error structure.
 546  *
 547  * @return
 548  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 549  */
 550 static int
 551 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 552                  const struct rte_flow_item items[],
 553                  const struct rte_flow_action actions[],
 554                  struct rte_flow_error *error)
 555 {
 556         int ret;
 557         uint64_t action_flags = 0;
 558         uint64_t item_flags = 0;
 559         int tunnel = 0;
 560         uint8_t next_protocol = 0xff;
 561         int actions_n = 0;
 562
 563         if (items == NULL)
 564                 return -1;
 565         ret = flow_dv_validate_attributes(dev, attr, error);
 566         if (ret < 0)
 567                 return ret;
 568         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 569                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 570                 switch (items->type) {
 571                 case RTE_FLOW_ITEM_TYPE_VOID:
 572                         break;
 573                 case RTE_FLOW_ITEM_TYPE_ETH:
 574                         ret = mlx5_flow_validate_item_eth(items, item_flags,
 575                                                           error);
 576                         if (ret < 0)
 577                                 return ret;
 578                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 579                                                MLX5_FLOW_LAYER_OUTER_L2;
 580                         break;
 581                 case RTE_FLOW_ITEM_TYPE_VLAN:
 582                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
 583                                                            error);
 584                         if (ret < 0)
 585                                 return ret;
 586                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
 587                                                MLX5_FLOW_LAYER_OUTER_VLAN;
 588                         break;
 589                 case RTE_FLOW_ITEM_TYPE_IPV4:
 590                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
 591                                                            error);
 592                         if (ret < 0)
 593                                 return ret;
 594                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 595                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 596                         if (items->mask != NULL &&
 597                             ((const struct rte_flow_item_ipv4 *)
 598                              items->mask)->hdr.next_proto_id)
 599                                 next_protocol =
 600                                         ((const struct rte_flow_item_ipv4 *)
 601                                          (items->spec))->hdr.next_proto_id;
 602                         break;
 603                 case RTE_FLOW_ITEM_TYPE_IPV6:
 604                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
 605                                                            error);
 606                         if (ret < 0)
 607                                 return ret;
 608                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 609                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 610                         if (items->mask != NULL &&
 611                             ((const struct rte_flow_item_ipv6 *)
 612                              items->mask)->hdr.proto)
 613                                 next_protocol =
 614                                         ((const struct rte_flow_item_ipv6 *)
 615                                          items->spec)->hdr.proto;
 616                         break;
 617                 case RTE_FLOW_ITEM_TYPE_TCP:
 618                         ret = mlx5_flow_validate_item_tcp
 619                                                 (items, item_flags,
 620                                                  next_protocol,
 621                                                  &rte_flow_item_tcp_mask,
 622                                                  error);
 623                         if (ret < 0)
 624                                 return ret;
 625                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 626                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
 627                         break;
 628                 case RTE_FLOW_ITEM_TYPE_UDP:
 629                         ret = mlx5_flow_validate_item_udp(items, item_flags,
 630                                                           next_protocol,
 631                                                           error);
 632                         if (ret < 0)
 633                                 return ret;
 634                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 635                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
 636                         break;
 637                 case RTE_FLOW_ITEM_TYPE_GRE:
 638                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 639                         ret = mlx5_flow_validate_item_gre(items, item_flags,
 640                                                           next_protocol, error);
 641                         if (ret < 0)
 642                                 return ret;
 643                         item_flags |= MLX5_FLOW_LAYER_GRE;
 644                         break;
 645                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 646                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
 647                                                             error);
 648                         if (ret < 0)
 649                                 return ret;
 650                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
 651                         break;
 652                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 653                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
 654                                                                 item_flags, dev,
 655                                                                 error);
 656                         if (ret < 0)
 657                                 return ret;
 658                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 659                         break;
 660                 case RTE_FLOW_ITEM_TYPE_META:
 661                         ret = flow_dv_validate_item_meta(dev, items, attr,
 662                                                          error);
 663                         if (ret < 0)
 664                                 return ret;
 665                         item_flags |= MLX5_FLOW_ITEM_METADATA;
 666                         break;
 667                 default:
 668                         return rte_flow_error_set(error, ENOTSUP,
 669                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 670                                                   NULL, "item not supported");
 671                 }
 672         }
 673         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 674                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
 675                         return rte_flow_error_set(error, ENOTSUP,
 676                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 677                                                   actions, "too many actions");
 678                 switch (actions->type) {
 679                 case RTE_FLOW_ACTION_TYPE_VOID:
 680                         break;
 681                 case RTE_FLOW_ACTION_TYPE_FLAG:
 682                         ret = mlx5_flow_validate_action_flag(action_flags,
 683                                                              attr, error);
 684                         if (ret < 0)
 685                                 return ret;
 686                         action_flags |= MLX5_FLOW_ACTION_FLAG;
 687                         ++actions_n;
 688                         break;
 689                 case RTE_FLOW_ACTION_TYPE_MARK:
 690                         ret = mlx5_flow_validate_action_mark(actions,
 691                                                              action_flags,
 692                                                              attr, error);
 693                         if (ret < 0)
 694                                 return ret;
 695                         action_flags |= MLX5_FLOW_ACTION_MARK;
 696                         ++actions_n;
 697                         break;
 698                 case RTE_FLOW_ACTION_TYPE_DROP:
 699                         ret = mlx5_flow_validate_action_drop(action_flags,
 700                                                              attr, error);
 701                         if (ret < 0)
 702                                 return ret;
 703                         action_flags |= MLX5_FLOW_ACTION_DROP;
 704                         ++actions_n;
 705                         break;
 706                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 707                         ret = mlx5_flow_validate_action_queue(actions,
 708                                                               action_flags, dev,
 709                                                               attr, error);
 710                         if (ret < 0)
 711                                 return ret;
 712                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
 713                         ++actions_n;
 714                         break;
 715                 case RTE_FLOW_ACTION_TYPE_RSS:
 716                         ret = mlx5_flow_validate_action_rss(actions,
 717                                                             action_flags, dev,
 718                                                             attr, error);
 719                         if (ret < 0)
 720                                 return ret;
 721                         action_flags |= MLX5_FLOW_ACTION_RSS;
 722                         ++actions_n;
 723                         break;
 724                 case RTE_FLOW_ACTION_TYPE_COUNT:
 725                         ret = mlx5_flow_validate_action_count(dev, attr, error);
 726                         if (ret < 0)
 727                                 return ret;
 728                         action_flags |= MLX5_FLOW_ACTION_COUNT;
 729                         ++actions_n;
 730                         break;
 731                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 732                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 733                         ret = flow_dv_validate_action_l2_encap(action_flags,
 734                                                                actions, attr,
 735                                                                error);
 736                         if (ret < 0)
 737                                 return ret;
 738                         action_flags |= actions->type ==
 739                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
 740                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
 741                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
 742
 743                         ++actions_n;
 744                         break;
 745                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
 746                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
 747                         ret = flow_dv_validate_action_l2_decap(action_flags,
 748                                                                attr, error);
 749                         if (ret < 0)
 750                                 return ret;
 751                         action_flags |= actions->type ==
 752                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
 753                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
 754                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
 755
 756                         ++actions_n;
 757                         break;
 758                 default:
 759                         return rte_flow_error_set(error, ENOTSUP,
 760                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 761                                                   actions,
 762                                                   "action not supported");
 763                 }
 764         }
 765         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
 766                 return rte_flow_error_set(error, EINVAL,
 767                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
 768                                           "no fate action is found");
 769         return 0;
 770 }
 771
 772 /**
 773  * Internal preparation function. Allocates the DV flow size,
 774  * this size is constant.
 775  *
 776  * @param[in] attr
 777  *   Pointer to the flow attributes.
 778  * @param[in] items
 779  *   Pointer to the list of items.
 780  * @param[in] actions
 781  *   Pointer to the list of actions.
 782  * @param[out] item_flags
 783  *   Pointer to bit mask of all items detected.
 784  * @param[out] action_flags
 785  *   Pointer to bit mask of all actions detected.
 786  * @param[out] error
 787  *   Pointer to the error structure.
 788  *
 789  * @return
 790  *   Pointer to mlx5_flow object on success,
 791  *   otherwise NULL and rte_ernno is set.
 792  */
 793 static struct mlx5_flow *
 794 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
 795                 const struct rte_flow_item items[] __rte_unused,
 796                 const struct rte_flow_action actions[] __rte_unused,
 797                 uint64_t *item_flags __rte_unused,
 798                 uint64_t *action_flags __rte_unused,
 799                 struct rte_flow_error *error)
 800 {
 801         uint32_t size = sizeof(struct mlx5_flow);
 802         struct mlx5_flow *flow;
 803
 804         flow = rte_calloc(__func__, 1, size, 0);
 805         if (!flow) {
 806                 rte_flow_error_set(error, ENOMEM,
 807                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 808                                    "not enough memory to create flow");
 809                 return NULL;
 810         }
 811         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
 812         return flow;
 813 }
 814
 815 /**
 816  * Add Ethernet item to matcher and to the value.
 817  *
 818  * @param[in, out] matcher
 819  *   Flow matcher.
 820  * @param[in, out] key
 821  *   Flow matcher value.
 822  * @param[in] item
 823  *   Flow pattern to translate.
 824  * @param[in] inner
 825  *   Item is inner pattern.
 826  */
 827 static void
 828 flow_dv_translate_item_eth(void *matcher, void *key,
 829                            const struct rte_flow_item *item, int inner)
 830 {
 831         const struct rte_flow_item_eth *eth_m = item->mask;
 832         const struct rte_flow_item_eth *eth_v = item->spec;
 833         const struct rte_flow_item_eth nic_mask = {
 834                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 835                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 836                 .type = RTE_BE16(0xffff),
 837         };
 838         void *headers_m;
 839         void *headers_v;
 840         char *l24_v;
 841         unsigned int i;
 842
 843         if (!eth_v)
 844                 return;
 845         if (!eth_m)
 846                 eth_m = &nic_mask;
 847         if (inner) {
 848                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 849                                          inner_headers);
 850                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 851         } else {
 852                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 853                                          outer_headers);
 854                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 855         }
 856         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
 857                &eth_m->dst, sizeof(eth_m->dst));
 858         /* The value must be in the range of the mask. */
 859         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
 860         for (i = 0; i < sizeof(eth_m->dst); ++i)
 861                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
 862         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
 863                &eth_m->src, sizeof(eth_m->src));
 864         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
 865         /* The value must be in the range of the mask. */
 866         for (i = 0; i < sizeof(eth_m->dst); ++i)
 867                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
 868         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
 869                  rte_be_to_cpu_16(eth_m->type));
 870         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
 871         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
 872 }
 873
 874 /**
 875  * Add VLAN item to matcher and to the value.
 876  *
 877  * @param[in, out] matcher
 878  *   Flow matcher.
 879  * @param[in, out] key
 880  *   Flow matcher value.
 881  * @param[in] item
 882  *   Flow pattern to translate.
 883  * @param[in] inner
 884  *   Item is inner pattern.
 885  */
 886 static void
 887 flow_dv_translate_item_vlan(void *matcher, void *key,
 888                             const struct rte_flow_item *item,
 889                             int inner)
 890 {
 891         const struct rte_flow_item_vlan *vlan_m = item->mask;
 892         const struct rte_flow_item_vlan *vlan_v = item->spec;
 893         const struct rte_flow_item_vlan nic_mask = {
 894                 .tci = RTE_BE16(0x0fff),
 895                 .inner_type = RTE_BE16(0xffff),
 896         };
 897         void *headers_m;
 898         void *headers_v;
 899         uint16_t tci_m;
 900         uint16_t tci_v;
 901
 902         if (!vlan_v)
 903                 return;
 904         if (!vlan_m)
 905                 vlan_m = &nic_mask;
 906         if (inner) {
 907                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 908                                          inner_headers);
 909                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 910         } else {
 911                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 912                                          outer_headers);
 913                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 914         }
 915         tci_m = rte_be_to_cpu_16(vlan_m->tci);
 916         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
 917         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
 918         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
 919         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
 920         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
 921         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
 922         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
 923         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
 924         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
 925 }
 926
 927 /**
 928  * Add IPV4 item to matcher and to the value.
 929  *
 930  * @param[in, out] matcher
 931  *   Flow matcher.
 932  * @param[in, out] key
 933  *   Flow matcher value.
 934  * @param[in] item
 935  *   Flow pattern to translate.
 936  * @param[in] inner
 937  *   Item is inner pattern.
 938  */
 939 static void
 940 flow_dv_translate_item_ipv4(void *matcher, void *key,
 941                             const struct rte_flow_item *item,
 942                             int inner)
 943 {
 944         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
 945         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
 946         const struct rte_flow_item_ipv4 nic_mask = {
 947                 .hdr = {
 948                         .src_addr = RTE_BE32(0xffffffff),
 949                         .dst_addr = RTE_BE32(0xffffffff),
 950                         .type_of_service = 0xff,
 951                         .next_proto_id = 0xff,
 952                 },
 953         };
 954         void *headers_m;
 955         void *headers_v;
 956         char *l24_m;
 957         char *l24_v;
 958         uint8_t tos;
 959
 960         if (inner) {
 961                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 962                                          inner_headers);
 963                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 964         } else {
 965                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 966                                          outer_headers);
 967                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 968         }
 969         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 970         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
 971         if (!ipv4_v)
 972                 return;
 973         if (!ipv4_m)
 974                 ipv4_m = &nic_mask;
 975         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 976                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 977         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 978                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 979         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
 980         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
 981         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 982                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 983         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 984                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 985         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
 986         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
 987         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
 988         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
 989                  ipv4_m->hdr.type_of_service);
 990         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
 991         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
 992                  ipv4_m->hdr.type_of_service >> 2);
 993         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
 994         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 995                  ipv4_m->hdr.next_proto_id);
 996         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 997                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
 998 }
 999
1000 /**
1001  * Add IPV6 item to matcher and to the value.
1002  *
1003  * @param[in, out] matcher
1004  *   Flow matcher.
1005  * @param[in, out] key
1006  *   Flow matcher value.
1007  * @param[in] item
1008  *   Flow pattern to translate.
1009  * @param[in] inner
1010  *   Item is inner pattern.
1011  */
1012 static void
1013 flow_dv_translate_item_ipv6(void *matcher, void *key,
1014                             const struct rte_flow_item *item,
1015                             int inner)
1016 {
1017         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
1018         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
1019         const struct rte_flow_item_ipv6 nic_mask = {
1020                 .hdr = {
1021                         .src_addr =
1022                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1023                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1024                         .dst_addr =
1025                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1026                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1027                         .vtc_flow = RTE_BE32(0xffffffff),
1028                         .proto = 0xff,
1029                         .hop_limits = 0xff,
1030                 },
1031         };
1032         void *headers_m;
1033         void *headers_v;
1034         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1035         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1036         char *l24_m;
1037         char *l24_v;
1038         uint32_t vtc_m;
1039         uint32_t vtc_v;
1040         int i;
1041         int size;
1042
1043         if (inner) {
1044                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1045                                          inner_headers);
1046                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1047         } else {
1048                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1049                                          outer_headers);
1050                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1051         }
1052         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
1053         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
1054         if (!ipv6_v)
1055                 return;
1056         if (!ipv6_m)
1057                 ipv6_m = &nic_mask;
1058         size = sizeof(ipv6_m->hdr.dst_addr);
1059         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1060                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1061         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1062                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1063         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
1064         for (i = 0; i < size; ++i)
1065                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
1066         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1067                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1068         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1069                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1070         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
1071         for (i = 0; i < size; ++i)
1072                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
1073         /* TOS. */
1074         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
1075         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
1076         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
1077         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
1078         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
1079         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
1080         /* Label. */
1081         if (inner) {
1082                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
1083                          vtc_m);
1084                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
1085                          vtc_v);
1086         } else {
1087                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
1088                          vtc_m);
1089                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
1090                          vtc_v);
1091         }
1092         /* Protocol. */
1093         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
1094                  ipv6_m->hdr.proto);
1095         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1096                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
1097 }
1098
1099 /**
1100  * Add TCP item to matcher and to the value.
1101  *
1102  * @param[in, out] matcher
1103  *   Flow matcher.
1104  * @param[in, out] key
1105  *   Flow matcher value.
1106  * @param[in] item
1107  *   Flow pattern to translate.
1108  * @param[in] inner
1109  *   Item is inner pattern.
1110  */
1111 static void
1112 flow_dv_translate_item_tcp(void *matcher, void *key,
1113                            const struct rte_flow_item *item,
1114                            int inner)
1115 {
1116         const struct rte_flow_item_tcp *tcp_m = item->mask;
1117         const struct rte_flow_item_tcp *tcp_v = item->spec;
1118         void *headers_m;
1119         void *headers_v;
1120
1121         if (inner) {
1122                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1123                                          inner_headers);
1124                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1125         } else {
1126                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1127                                          outer_headers);
1128                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1129         }
1130         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1131         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
1132         if (!tcp_v)
1133                 return;
1134         if (!tcp_m)
1135                 tcp_m = &rte_flow_item_tcp_mask;
1136         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
1137                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
1138         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
1139                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
1140         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
1141                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
1142         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
1143                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
1144 }
1145
1146 /**
1147  * Add UDP item to matcher and to the value.
1148  *
1149  * @param[in, out] matcher
1150  *   Flow matcher.
1151  * @param[in, out] key
1152  *   Flow matcher value.
1153  * @param[in] item
1154  *   Flow pattern to translate.
1155  * @param[in] inner
1156  *   Item is inner pattern.
1157  */
1158 static void
1159 flow_dv_translate_item_udp(void *matcher, void *key,
1160                            const struct rte_flow_item *item,
1161                            int inner)
1162 {
1163         const struct rte_flow_item_udp *udp_m = item->mask;
1164         const struct rte_flow_item_udp *udp_v = item->spec;
1165         void *headers_m;
1166         void *headers_v;
1167
1168         if (inner) {
1169                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1170                                          inner_headers);
1171                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1172         } else {
1173                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1174                                          outer_headers);
1175                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1176         }
1177         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1178         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
1179         if (!udp_v)
1180                 return;
1181         if (!udp_m)
1182                 udp_m = &rte_flow_item_udp_mask;
1183         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
1184                  rte_be_to_cpu_16(udp_m->hdr.src_port));
1185         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
1186                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
1187         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
1188                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
1189         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
1190                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
1191 }
1192
1193 /**
1194  * Add GRE item to matcher and to the value.
1195  *
1196  * @param[in, out] matcher
1197  *   Flow matcher.
1198  * @param[in, out] key
1199  *   Flow matcher value.
1200  * @param[in] item
1201  *   Flow pattern to translate.
1202  * @param[in] inner
1203  *   Item is inner pattern.
1204  */
1205 static void
1206 flow_dv_translate_item_gre(void *matcher, void *key,
1207                            const struct rte_flow_item *item,
1208                            int inner)
1209 {
1210         const struct rte_flow_item_gre *gre_m = item->mask;
1211         const struct rte_flow_item_gre *gre_v = item->spec;
1212         void *headers_m;
1213         void *headers_v;
1214         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1215         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1216
1217         if (inner) {
1218                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1219                                          inner_headers);
1220                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1221         } else {
1222                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1223                                          outer_headers);
1224                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1225         }
1226         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1227         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
1228         if (!gre_v)
1229                 return;
1230         if (!gre_m)
1231                 gre_m = &rte_flow_item_gre_mask;
1232         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
1233                  rte_be_to_cpu_16(gre_m->protocol));
1234         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
1235                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
1236 }
1237
1238 /**
1239  * Add NVGRE item to matcher and to the value.
1240  *
1241  * @param[in, out] matcher
1242  *   Flow matcher.
1243  * @param[in, out] key
1244  *   Flow matcher value.
1245  * @param[in] item
1246  *   Flow pattern to translate.
1247  * @param[in] inner
1248  *   Item is inner pattern.
1249  */
1250 static void
1251 flow_dv_translate_item_nvgre(void *matcher, void *key,
1252                              const struct rte_flow_item *item,
1253                              int inner)
1254 {
1255         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
1256         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
1257         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1258         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1259         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
1260         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
1261         char *gre_key_m;
1262         char *gre_key_v;
1263         int size;
1264         int i;
1265
1266         flow_dv_translate_item_gre(matcher, key, item, inner);
1267         if (!nvgre_v)
1268                 return;
1269         if (!nvgre_m)
1270                 nvgre_m = &rte_flow_item_nvgre_mask;
1271         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
1272         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
1273         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
1274         memcpy(gre_key_m, tni_flow_id_m, size);
1275         for (i = 0; i < size; ++i)
1276                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
1277 }
1278
1279 /**
1280  * Add VXLAN item to matcher and to the value.
1281  *
1282  * @param[in, out] matcher
1283  *   Flow matcher.
1284  * @param[in, out] key
1285  *   Flow matcher value.
1286  * @param[in] item
1287  *   Flow pattern to translate.
1288  * @param[in] inner
1289  *   Item is inner pattern.
1290  */
1291 static void
1292 flow_dv_translate_item_vxlan(void *matcher, void *key,
1293                              const struct rte_flow_item *item,
1294                              int inner)
1295 {
1296         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
1297         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
1298         void *headers_m;
1299         void *headers_v;
1300         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1301         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1302         char *vni_m;
1303         char *vni_v;
1304         uint16_t dport;
1305         int size;
1306         int i;
1307
1308         if (inner) {
1309                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1310                                          inner_headers);
1311                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1312         } else {
1313                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1314                                          outer_headers);
1315                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1316         }
1317         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
1318                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
1319         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
1320                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
1321                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
1322         }
1323         if (!vxlan_v)
1324                 return;
1325         if (!vxlan_m)
1326                 vxlan_m = &rte_flow_item_vxlan_mask;
1327         size = sizeof(vxlan_m->vni);
1328         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
1329         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
1330         memcpy(vni_m, vxlan_m->vni, size);
1331         for (i = 0; i < size; ++i)
1332                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
1333 }
1334
1335 /**
1336  * Add META item to matcher
1337  *
1338  * @param[in, out] matcher
1339  *   Flow matcher.
1340  * @param[in, out] key
1341  *   Flow matcher value.
1342  * @param[in] item
1343  *   Flow pattern to translate.
1344  * @param[in] inner
1345  *   Item is inner pattern.
1346  */
1347 static void
1348 flow_dv_translate_item_meta(void *matcher, void *key,
1349                             const struct rte_flow_item *item)
1350 {
1351         const struct rte_flow_item_meta *meta_m;
1352         const struct rte_flow_item_meta *meta_v;
1353         void *misc2_m =
1354                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
1355         void *misc2_v =
1356                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
1357
1358         meta_m = (const void *)item->mask;
1359         if (!meta_m)
1360                 meta_m = &rte_flow_item_meta_mask;
1361         meta_v = (const void *)item->spec;
1362         if (meta_v) {
1363                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
1364                          rte_be_to_cpu_32(meta_m->data));
1365                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
1366                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
1367         }
1368 }
1369
1370 /**
1371  * Update the matcher and the value based the selected item.
1372  *
1373  * @param[in, out] matcher
1374  *   Flow matcher.
1375  * @param[in, out] key
1376  *   Flow matcher value.
1377  * @param[in] item
1378  *   Flow pattern to translate.
1379  * @param[in, out] dev_flow
1380  *   Pointer to the mlx5_flow.
1381  * @param[in] inner
1382  *   Item is inner pattern.
1383  */
1384 static void
1385 flow_dv_create_item(void *matcher, void *key,
1386                     const struct rte_flow_item *item,
1387                     struct mlx5_flow *dev_flow,
1388                     int inner)
1389 {
1390         struct mlx5_flow_dv_matcher *tmatcher = matcher;
1391
1392         switch (item->type) {
1393         case RTE_FLOW_ITEM_TYPE_ETH:
1394                 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
1395                                            inner);
1396                 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
1397                 break;
1398         case RTE_FLOW_ITEM_TYPE_VLAN:
1399                 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
1400                                             inner);
1401                 break;
1402         case RTE_FLOW_ITEM_TYPE_IPV4:
1403                 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
1404                                             inner);
1405                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1406                 dev_flow->dv.hash_fields |=
1407                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1408                                                     MLX5_IPV4_LAYER_TYPES,
1409                                                     MLX5_IPV4_IBV_RX_HASH);
1410                 break;
1411         case RTE_FLOW_ITEM_TYPE_IPV6:
1412                 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
1413                                             inner);
1414                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1415                 dev_flow->dv.hash_fields |=
1416                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1417                                                     MLX5_IPV6_LAYER_TYPES,
1418                                                     MLX5_IPV6_IBV_RX_HASH);
1419                 break;
1420         case RTE_FLOW_ITEM_TYPE_TCP:
1421                 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
1422                                            inner);
1423                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1424                 dev_flow->dv.hash_fields |=
1425                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1426                                                     ETH_RSS_TCP,
1427                                                     (IBV_RX_HASH_SRC_PORT_TCP |
1428                                                      IBV_RX_HASH_DST_PORT_TCP));
1429                 break;
1430         case RTE_FLOW_ITEM_TYPE_UDP:
1431                 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
1432                                            inner);
1433                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1434                 dev_flow->verbs.hash_fields |=
1435                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1436                                                     ETH_RSS_UDP,
1437                                                     (IBV_RX_HASH_SRC_PORT_UDP |
1438                                                      IBV_RX_HASH_DST_PORT_UDP));
1439                 break;
1440         case RTE_FLOW_ITEM_TYPE_GRE:
1441                 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
1442                                            inner);
1443                 break;
1444         case RTE_FLOW_ITEM_TYPE_NVGRE:
1445                 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
1446                                              inner);
1447                 break;
1448         case RTE_FLOW_ITEM_TYPE_VXLAN:
1449         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1450                 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
1451                                              inner);
1452                 break;
1453         case RTE_FLOW_ITEM_TYPE_META:
1454                 flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
1455                 break;
1456         default:
1457                 break;
1458         }
1459 }
1460
1461 /**
1462  * Store the requested actions in an array.
1463  *
1464  * @param[in] dev
1465  *   Pointer to rte_eth_dev structure.
1466  * @param[in] action
1467  *   Flow action to translate.
1468  * @param[in, out] dev_flow
1469  *   Pointer to the mlx5_flow.
1470  * @param[out] error
1471  *   Pointer to the error structure.
1472  *
1473  * @return
1474  *   0 on success, a negative errno value otherwise and rte_errno is set.
1475  */
1476 static int
1477 flow_dv_create_action(struct rte_eth_dev *dev,
1478                       const struct rte_flow_action *action,
1479                       struct mlx5_flow *dev_flow,
1480                       struct rte_flow_error *error)
1481 {
1482         const struct rte_flow_action_queue *queue;
1483         const struct rte_flow_action_rss *rss;
1484         int actions_n = dev_flow->dv.actions_n;
1485         struct rte_flow *flow = dev_flow->flow;
1486
1487         switch (action->type) {
1488         case RTE_FLOW_ACTION_TYPE_VOID:
1489                 break;
1490         case RTE_FLOW_ACTION_TYPE_FLAG:
1491                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1492                 dev_flow->dv.actions[actions_n].tag_value =
1493                         mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
1494                 actions_n++;
1495                 flow->actions |= MLX5_FLOW_ACTION_FLAG;
1496                 break;
1497         case RTE_FLOW_ACTION_TYPE_MARK:
1498                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1499                 dev_flow->dv.actions[actions_n].tag_value =
1500                         mlx5_flow_mark_set
1501                         (((const struct rte_flow_action_mark *)
1502                           (action->conf))->id);
1503                 flow->actions |= MLX5_FLOW_ACTION_MARK;
1504                 actions_n++;
1505                 break;
1506         case RTE_FLOW_ACTION_TYPE_DROP:
1507                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
1508                 flow->actions |= MLX5_FLOW_ACTION_DROP;
1509                 break;
1510         case RTE_FLOW_ACTION_TYPE_QUEUE:
1511                 queue = action->conf;
1512                 flow->rss.queue_num = 1;
1513                 (*flow->queue)[0] = queue->index;
1514                 flow->actions |= MLX5_FLOW_ACTION_QUEUE;
1515                 break;
1516         case RTE_FLOW_ACTION_TYPE_RSS:
1517                 rss = action->conf;
1518                 if (flow->queue)
1519                         memcpy((*flow->queue), rss->queue,
1520                                rss->queue_num * sizeof(uint16_t));
1521                 flow->rss.queue_num = rss->queue_num;
1522                 memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
1523                 flow->rss.types = rss->types;
1524                 flow->rss.level = rss->level;
1525                 /* Added to array only in apply since we need the QP */
1526                 flow->actions |= MLX5_FLOW_ACTION_RSS;
1527                 break;
1528         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1529         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
1530                 dev_flow->dv.actions[actions_n].type =
1531                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1532                 dev_flow->dv.actions[actions_n].action =
1533                                 flow_dv_create_action_l2_encap(dev, action,
1534                                                                error);
1535                 if (!(dev_flow->dv.actions[actions_n].action))
1536                         return -rte_errno;
1537                 dev_flow->dv.encap_decap_verbs_action =
1538                         dev_flow->dv.actions[actions_n].action;
1539                 flow->actions |= action->type ==
1540                                  RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
1541                                  MLX5_FLOW_ACTION_VXLAN_ENCAP :
1542                                  MLX5_FLOW_ACTION_NVGRE_ENCAP;
1543                 actions_n++;
1544                 break;
1545         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1546         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
1547                 dev_flow->dv.actions[actions_n].type =
1548                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1549                 dev_flow->dv.actions[actions_n].action =
1550                                 flow_dv_create_action_l2_decap(dev, error);
1551                 if (!(dev_flow->dv.actions[actions_n].action))
1552                         return -rte_errno;
1553                 dev_flow->dv.encap_decap_verbs_action =
1554                         dev_flow->dv.actions[actions_n].action;
1555                 flow->actions |= action->type ==
1556                                  RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
1557                                  MLX5_FLOW_ACTION_VXLAN_DECAP :
1558                                  MLX5_FLOW_ACTION_NVGRE_DECAP;
1559                 actions_n++;
1560                 break;
1561         default:
1562                 break;
1563         }
1564         dev_flow->dv.actions_n = actions_n;
1565         return 0;
1566 }
1567
1568 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
1569
1570 #define HEADER_IS_ZERO(match_criteria, headers)                              \
1571         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
1572                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1573
1574 /**
1575  * Calculate flow matcher enable bitmap.
1576  *
1577  * @param match_criteria
1578  *   Pointer to flow matcher criteria.
1579  *
1580  * @return
1581  *   Bitmap of enabled fields.
1582  */
1583 static uint8_t
1584 flow_dv_matcher_enable(uint32_t *match_criteria)
1585 {
1586         uint8_t match_criteria_enable;
1587
1588         match_criteria_enable =
1589                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1590                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
1591         match_criteria_enable |=
1592                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1593                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
1594         match_criteria_enable |=
1595                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1596                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
1597         match_criteria_enable |=
1598                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
1599                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
1600
1601         return match_criteria_enable;
1602 }
1603
1604 /**
1605  * Register the flow matcher.
1606  *
1607  * @param dev[in, out]
1608  *   Pointer to rte_eth_dev structure.
1609  * @param[in, out] matcher
1610  *   Pointer to flow matcher.
1611  * @parm[in, out] dev_flow
1612  *   Pointer to the dev_flow.
1613  * @param[out] error
1614  *   pointer to error structure.
1615  *
1616  * @return
1617  *   0 on success otherwise -errno and errno is set.
1618  */
1619 static int
1620 flow_dv_matcher_register(struct rte_eth_dev *dev,
1621                          struct mlx5_flow_dv_matcher *matcher,
1622                          struct mlx5_flow *dev_flow,
1623                          struct rte_flow_error *error)
1624 {
1625         struct priv *priv = dev->data->dev_private;
1626         struct mlx5_flow_dv_matcher *cache_matcher;
1627         struct mlx5dv_flow_matcher_attr dv_attr = {
1628                 .type = IBV_FLOW_ATTR_NORMAL,
1629                 .match_mask = (void *)&matcher->mask,
1630         };
1631
1632         /* Lookup from cache. */
1633         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1634                 if (matcher->crc == cache_matcher->crc &&
1635                     matcher->priority == cache_matcher->priority &&
1636                     matcher->egress == cache_matcher->egress &&
1637                     !memcmp((const void *)matcher->mask.buf,
1638                             (const void *)cache_matcher->mask.buf,
1639                             cache_matcher->mask.size)) {
1640                         DRV_LOG(DEBUG,
1641                                 "priority %hd use %s matcher %p: refcnt %d++",
1642                                 cache_matcher->priority,
1643                                 cache_matcher->egress ? "tx" : "rx",
1644                                 (void *)cache_matcher,
1645                                 rte_atomic32_read(&cache_matcher->refcnt));
1646                         rte_atomic32_inc(&cache_matcher->refcnt);
1647                         dev_flow->dv.matcher = cache_matcher;
1648                         return 0;
1649                 }
1650         }
1651         /* Register new matcher. */
1652         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1653         if (!cache_matcher)
1654                 return rte_flow_error_set(error, ENOMEM,
1655                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1656                                           "cannot allocate matcher memory");
1657         *cache_matcher = *matcher;
1658         dv_attr.match_criteria_enable =
1659                 flow_dv_matcher_enable(cache_matcher->mask.buf);
1660         dv_attr.priority = matcher->priority;
1661         if (matcher->egress)
1662                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1663         cache_matcher->matcher_object =
1664                 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
1665         if (!cache_matcher->matcher_object) {
1666                 rte_free(cache_matcher);
1667                 return rte_flow_error_set(error, ENOMEM,
1668                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1669                                           NULL, "cannot create matcher");
1670         }
1671         rte_atomic32_inc(&cache_matcher->refcnt);
1672         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1673         dev_flow->dv.matcher = cache_matcher;
1674         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1675                 cache_matcher->priority,
1676                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1677                 rte_atomic32_read(&cache_matcher->refcnt));
1678         return 0;
1679 }
1680
1681 /**
1682  * Fill the flow with DV spec.
1683  *
1684  * @param[in] dev
1685  *   Pointer to rte_eth_dev structure.
1686  * @param[in, out] dev_flow
1687  *   Pointer to the sub flow.
1688  * @param[in] attr
1689  *   Pointer to the flow attributes.
1690  * @param[in] items
1691  *   Pointer to the list of items.
1692  * @param[in] actions
1693  *   Pointer to the list of actions.
1694  * @param[out] error
1695  *   Pointer to the error structure.
1696  *
1697  * @return
1698  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1699  */
1700 static int
1701 flow_dv_translate(struct rte_eth_dev *dev,
1702                   struct mlx5_flow *dev_flow,
1703                   const struct rte_flow_attr *attr,
1704                   const struct rte_flow_item items[],
1705                   const struct rte_flow_action actions[] __rte_unused,
1706                   struct rte_flow_error *error)
1707 {
1708         struct priv *priv = dev->data->dev_private;
1709         uint64_t priority = attr->priority;
1710         struct mlx5_flow_dv_matcher matcher = {
1711                 .mask = {
1712                         .size = sizeof(matcher.mask.buf),
1713                 },
1714         };
1715         void *match_value = dev_flow->dv.value.buf;
1716         int tunnel = 0;
1717
1718         if (priority == MLX5_FLOW_PRIO_RSVD)
1719                 priority = priv->config.flow_prio - 1;
1720         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1721                 tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
1722                 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1723                                     tunnel);
1724         }
1725         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1726                                      matcher.mask.size);
1727         if (priority == MLX5_FLOW_PRIO_RSVD)
1728                 priority = priv->config.flow_prio - 1;
1729         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1730                                                      matcher.priority);
1731         matcher.egress = attr->egress;
1732         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1733                 return -rte_errno;
1734         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1735                 if (flow_dv_create_action(dev, actions, dev_flow, error))
1736                         return -rte_errno;
1737         return 0;
1738 }
1739
1740 /**
1741  * Apply the flow to the NIC.
1742  *
1743  * @param[in] dev
1744  *   Pointer to the Ethernet device structure.
1745  * @param[in, out] flow
1746  *   Pointer to flow structure.
1747  * @param[out] error
1748  *   Pointer to error structure.
1749  *
1750  * @return
1751  *   0 on success, a negative errno value otherwise and rte_errno is set.
1752  */
1753 static int
1754 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1755               struct rte_flow_error *error)
1756 {
1757         struct mlx5_flow_dv *dv;
1758         struct mlx5_flow *dev_flow;
1759         int n;
1760         int err;
1761
1762         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1763                 dv = &dev_flow->dv;
1764                 n = dv->actions_n;
1765                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1766                         dv->hrxq = mlx5_hrxq_drop_new(dev);
1767                         if (!dv->hrxq) {
1768                                 rte_flow_error_set
1769                                         (error, errno,
1770                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1771                                          "cannot get drop hash queue");
1772                                 goto error;
1773                         }
1774                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1775                         dv->actions[n].qp = dv->hrxq->qp;
1776                         n++;
1777                 } else if (flow->actions &
1778                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
1779                         struct mlx5_hrxq *hrxq;
1780                         hrxq = mlx5_hrxq_get(dev, flow->key,
1781                                              MLX5_RSS_HASH_KEY_LEN,
1782                                              dv->hash_fields,
1783                                              (*flow->queue),
1784                                              flow->rss.queue_num);
1785                         if (!hrxq)
1786                                 hrxq = mlx5_hrxq_new
1787                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
1788                                          dv->hash_fields, (*flow->queue),
1789                                          flow->rss.queue_num,
1790                                          !!(dev_flow->layers &
1791                                             MLX5_FLOW_LAYER_TUNNEL));
1792                         if (!hrxq) {
1793                                 rte_flow_error_set
1794                                         (error, rte_errno,
1795                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1796                                          "cannot get hash queue");
1797                                 goto error;
1798                         }
1799                         dv->hrxq = hrxq;
1800                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1801                         dv->actions[n].qp = hrxq->qp;
1802                         n++;
1803                 }
1804                 dv->flow =
1805                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
1806                                                   (void *)&dv->value, n,
1807                                                   dv->actions);
1808                 if (!dv->flow) {
1809                         rte_flow_error_set(error, errno,
1810                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1811                                            NULL,
1812                                            "hardware refuses to create flow");
1813                         goto error;
1814                 }
1815         }
1816         return 0;
1817 error:
1818         err = rte_errno; /* Save rte_errno before cleanup. */
1819         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1820                 struct mlx5_flow_dv *dv = &dev_flow->dv;
1821                 if (dv->hrxq) {
1822                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1823                                 mlx5_hrxq_drop_release(dev);
1824                         else
1825                                 mlx5_hrxq_release(dev, dv->hrxq);
1826                         dv->hrxq = NULL;
1827                 }
1828         }
1829         rte_errno = err; /* Restore rte_errno. */
1830         return -rte_errno;
1831 }
1832
1833 /**
1834  * Release the flow matcher.
1835  *
1836  * @param dev
1837  *   Pointer to Ethernet device.
1838  * @param flow
1839  *   Pointer to mlx5_flow.
1840  *
1841  * @return
1842  *   1 while a reference on it exists, 0 when freed.
1843  */
1844 static int
1845 flow_dv_matcher_release(struct rte_eth_dev *dev,
1846                         struct mlx5_flow *flow)
1847 {
1848         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
1849
1850         assert(matcher->matcher_object);
1851         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
1852                 dev->data->port_id, (void *)matcher,
1853                 rte_atomic32_read(&matcher->refcnt));
1854         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
1855                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
1856                            (matcher->matcher_object));
1857                 LIST_REMOVE(matcher, next);
1858                 rte_free(matcher);
1859                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
1860                         dev->data->port_id, (void *)matcher);
1861                 return 0;
1862         }
1863         return 1;
1864 }
1865
1866 /**
1867  * Remove the flow from the NIC but keeps it in memory.
1868  *
1869  * @param[in] dev
1870  *   Pointer to Ethernet device.
1871  * @param[in, out] flow
1872  *   Pointer to flow structure.
1873  */
1874 static void
1875 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1876 {
1877         struct mlx5_flow_dv *dv;
1878         struct mlx5_flow *dev_flow;
1879
1880         if (!flow)
1881                 return;
1882         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1883                 dv = &dev_flow->dv;
1884                 if (dv->flow) {
1885                         claim_zero(mlx5_glue->destroy_flow(dv->flow));
1886                         dv->flow = NULL;
1887                 }
1888                 if (dv->hrxq) {
1889                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1890                                 mlx5_hrxq_drop_release(dev);
1891                         else
1892                                 mlx5_hrxq_release(dev, dv->hrxq);
1893                         dv->hrxq = NULL;
1894                 }
1895         }
1896         if (flow->counter)
1897                 flow->counter = NULL;
1898 }
1899
1900 /**
1901  * Remove the flow from the NIC and the memory.
1902  *
1903  * @param[in] dev
1904  *   Pointer to the Ethernet device structure.
1905  * @param[in, out] flow
1906  *   Pointer to flow structure.
1907  */
1908 static void
1909 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1910 {
1911         struct mlx5_flow *dev_flow;
1912
1913         if (!flow)
1914                 return;
1915         flow_dv_remove(dev, flow);
1916         while (!LIST_EMPTY(&flow->dev_flows)) {
1917                 dev_flow = LIST_FIRST(&flow->dev_flows);
1918                 LIST_REMOVE(dev_flow, next);
1919                 if (dev_flow->dv.matcher)
1920                         flow_dv_matcher_release(dev, dev_flow);
1921                 if (dev_flow->dv.encap_decap_verbs_action) {
1922                         claim_zero(mlx5_glue->destroy_flow_action
1923                                 (dev_flow->dv.encap_decap_verbs_action));
1924                         dev_flow->dv.encap_decap_verbs_action = NULL;
1925                 }
1926                 rte_free(dev_flow);
1927         }
1928 }
1929
1930 /**
1931  * Query a flow.
1932  *
1933  * @see rte_flow_query()
1934  * @see rte_flow_ops
1935  */
1936 static int
1937 flow_dv_query(struct rte_eth_dev *dev __rte_unused,
1938               struct rte_flow *flow __rte_unused,
1939               const struct rte_flow_action *actions __rte_unused,
1940               void *data __rte_unused,
1941               struct rte_flow_error *error __rte_unused)
1942 {
1943         rte_errno = ENOTSUP;
1944         return -rte_errno;
1945 }
1946
1947
1948 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
1949         .validate = flow_dv_validate,
1950         .prepare = flow_dv_prepare,
1951         .translate = flow_dv_translate,
1952         .apply = flow_dv_apply,
1953         .remove = flow_dv_remove,
1954         .destroy = flow_dv_destroy,
1955         .query = flow_dv_query,
1956 };
1957
1958 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */