drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  37
  38 #define MLX5_ENCAP_MAX_LEN 132
  39
  40 /**
  41  * Validate META item.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the rte_eth_dev structure.
  45  * @param[in] item
  46  *   Item specification.
  47  * @param[in] attr
  48  *   Attributes of flow that includes this item.
  49  * @param[out] error
  50  *   Pointer to error structure.
  51  *
  52  * @return
  53  *   0 on success, a negative errno value otherwise and rte_errno is set.
  54  */
  55 static int
  56 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
  57                            const struct rte_flow_item *item,
  58                            const struct rte_flow_attr *attr,
  59                            struct rte_flow_error *error)
  60 {
  61         const struct rte_flow_item_meta *spec = item->spec;
  62         const struct rte_flow_item_meta *mask = item->mask;
  63         const struct rte_flow_item_meta nic_mask = {
  64                 .data = RTE_BE32(UINT32_MAX)
  65         };
  66         int ret;
  67         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
  68
  69         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
  70                 return rte_flow_error_set(error, EPERM,
  71                                           RTE_FLOW_ERROR_TYPE_ITEM,
  72                                           NULL,
  73                                           "match on metadata offload "
  74                                           "configuration is off for this port");
  75         if (!spec)
  76                 return rte_flow_error_set(error, EINVAL,
  77                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  78                                           item->spec,
  79                                           "data cannot be empty");
  80         if (!spec->data)
  81                 return rte_flow_error_set(error, EINVAL,
  82                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  83                                           NULL,
  84                                           "data cannot be zero");
  85         if (!mask)
  86                 mask = &rte_flow_item_meta_mask;
  87         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
  88                                         (const uint8_t *)&nic_mask,
  89                                         sizeof(struct rte_flow_item_meta),
  90                                         error);
  91         if (ret < 0)
  92                 return ret;
  93         if (attr->ingress)
  94                 return rte_flow_error_set(error, ENOTSUP,
  95                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
  96                                           NULL,
  97                                           "pattern not supported for ingress");
  98         return 0;
  99 }
 100
 101 /**
 102  * Validate the L2 encap action.
 103  *
 104  * @param[in] action_flags
 105  *   Holds the actions detected until now.
 106  * @param[in] action
 107  *   Pointer to the encap action.
 108  * @param[in] attr
 109  *   Pointer to flow attributes
 110  * @param[out] error
 111  *   Pointer to error structure.
 112  *
 113  * @return
 114  *   0 on success, a negative errno value otherwise and rte_errno is set.
 115  */
 116 static int
 117 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 118                                  const struct rte_flow_action *action,
 119                                  const struct rte_flow_attr *attr,
 120                                  struct rte_flow_error *error)
 121 {
 122         if (!(action->conf))
 123                 return rte_flow_error_set(error, EINVAL,
 124                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 125                                           "configuration cannot be null");
 126         if (action_flags & MLX5_FLOW_ACTION_DROP)
 127                 return rte_flow_error_set(error, EINVAL,
 128                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 129                                           "can't drop and encap in same flow");
 130         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS |
 131                             MLX5_FLOW_ACTION_VXLAN_DECAP))
 132                 return rte_flow_error_set(error, EINVAL,
 133                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 134                                           "can only have a single encap or"
 135                                           " decap action in a flow");
 136         if (attr->ingress)
 137                 return rte_flow_error_set(error, ENOTSUP,
 138                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 139                                           NULL,
 140                                           "encap action not supported for "
 141                                           "ingress");
 142         return 0;
 143 }
 144
 145 /**
 146  * Validate the L2 decap action.
 147  *
 148  * @param[in] action_flags
 149  *   Holds the actions detected until now.
 150  * @param[in] attr
 151  *   Pointer to flow attributes
 152  * @param[out] error
 153  *   Pointer to error structure.
 154  *
 155  * @return
 156  *   0 on success, a negative errno value otherwise and rte_errno is set.
 157  */
 158 static int
 159 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 160                                  const struct rte_flow_attr *attr,
 161                                  struct rte_flow_error *error)
 162 {
 163         if (action_flags & MLX5_FLOW_ACTION_DROP)
 164                 return rte_flow_error_set(error, EINVAL,
 165                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 166                                           "can't drop and decap in same flow");
 167         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS |
 168                             MLX5_FLOW_ACTION_VXLAN_DECAP))
 169                 return rte_flow_error_set(error, EINVAL,
 170                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 171                                           "can only have a single encap or"
 172                                           " decap action in a flow");
 173         if (attr->egress)
 174                 return rte_flow_error_set(error, ENOTSUP,
 175                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 176                                           NULL,
 177                                           "decap action not supported for "
 178                                           "egress");
 179         return 0;
 180 }
 181
 182 /**
 183  * Get the size of specific rte_flow_item_type
 184  *
 185  * @param[in] item_type
 186  *   Tested rte_flow_item_type.
 187  *
 188  * @return
 189  *   sizeof struct item_type, 0 if void or irrelevant.
 190  */
 191 static size_t
 192 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
 193 {
 194         size_t retval;
 195
 196         switch (item_type) {
 197         case RTE_FLOW_ITEM_TYPE_ETH:
 198                 retval = sizeof(struct rte_flow_item_eth);
 199                 break;
 200         case RTE_FLOW_ITEM_TYPE_VLAN:
 201                 retval = sizeof(struct rte_flow_item_vlan);
 202                 break;
 203         case RTE_FLOW_ITEM_TYPE_IPV4:
 204                 retval = sizeof(struct rte_flow_item_ipv4);
 205                 break;
 206         case RTE_FLOW_ITEM_TYPE_IPV6:
 207                 retval = sizeof(struct rte_flow_item_ipv6);
 208                 break;
 209         case RTE_FLOW_ITEM_TYPE_UDP:
 210                 retval = sizeof(struct rte_flow_item_udp);
 211                 break;
 212         case RTE_FLOW_ITEM_TYPE_TCP:
 213                 retval = sizeof(struct rte_flow_item_tcp);
 214                 break;
 215         case RTE_FLOW_ITEM_TYPE_VXLAN:
 216                 retval = sizeof(struct rte_flow_item_vxlan);
 217                 break;
 218         case RTE_FLOW_ITEM_TYPE_GRE:
 219                 retval = sizeof(struct rte_flow_item_gre);
 220                 break;
 221         case RTE_FLOW_ITEM_TYPE_NVGRE:
 222                 retval = sizeof(struct rte_flow_item_nvgre);
 223                 break;
 224         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 225                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
 226                 break;
 227         case RTE_FLOW_ITEM_TYPE_MPLS:
 228                 retval = sizeof(struct rte_flow_item_mpls);
 229                 break;
 230         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
 231         default:
 232                 retval = 0;
 233                 break;
 234         }
 235         return retval;
 236 }
 237
 238 #define MLX5_ENCAP_IPV4_VERSION         0x40
 239 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
 240 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
 241 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
 242 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
 243 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
 244 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
 245
 246 /**
 247  * Convert the encap action data from list of rte_flow_item to raw buffer
 248  *
 249  * @param[in] items
 250  *   Pointer to rte_flow_item objects list.
 251  * @param[out] buf
 252  *   Pointer to the output buffer.
 253  * @param[out] size
 254  *   Pointer to the output buffer size.
 255  * @param[out] error
 256  *   Pointer to the error structure.
 257  *
 258  * @return
 259  *   0 on success, a negative errno value otherwise and rte_errno is set.
 260  */
 261 static int
 262 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 263                            size_t *size, struct rte_flow_error *error)
 264 {
 265         struct ether_hdr *eth = NULL;
 266         struct vlan_hdr *vlan = NULL;
 267         struct ipv4_hdr *ipv4 = NULL;
 268         struct ipv6_hdr *ipv6 = NULL;
 269         struct udp_hdr *udp = NULL;
 270         struct vxlan_hdr *vxlan = NULL;
 271         struct vxlan_gpe_hdr *vxlan_gpe = NULL;
 272         struct gre_hdr *gre = NULL;
 273         size_t len;
 274         size_t temp_size = 0;
 275
 276         if (!items)
 277                 return rte_flow_error_set(error, EINVAL,
 278                                           RTE_FLOW_ERROR_TYPE_ACTION,
 279                                           NULL, "invalid empty data");
 280         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 281                 len = flow_dv_get_item_len(items->type);
 282                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
 283                         return rte_flow_error_set(error, EINVAL,
 284                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 285                                                   (void *)items->type,
 286                                                   "items total size is too big"
 287                                                   " for encap action");
 288                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
 289                 switch (items->type) {
 290                 case RTE_FLOW_ITEM_TYPE_ETH:
 291                         eth = (struct ether_hdr *)&buf[temp_size];
 292                         break;
 293                 case RTE_FLOW_ITEM_TYPE_VLAN:
 294                         vlan = (struct vlan_hdr *)&buf[temp_size];
 295                         if (!eth)
 296                                 return rte_flow_error_set(error, EINVAL,
 297                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 298                                                 (void *)items->type,
 299                                                 "eth header not found");
 300                         if (!eth->ether_type)
 301                                 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
 302                         break;
 303                 case RTE_FLOW_ITEM_TYPE_IPV4:
 304                         ipv4 = (struct ipv4_hdr *)&buf[temp_size];
 305                         if (!vlan && !eth)
 306                                 return rte_flow_error_set(error, EINVAL,
 307                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 308                                                 (void *)items->type,
 309                                                 "neither eth nor vlan"
 310                                                 " header found");
 311                         if (vlan && !vlan->eth_proto)
 312                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
 313                         else if (eth && !eth->ether_type)
 314                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
 315                         if (!ipv4->version_ihl)
 316                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
 317                                                     MLX5_ENCAP_IPV4_IHL_MIN;
 318                         if (!ipv4->time_to_live)
 319                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
 320                         break;
 321                 case RTE_FLOW_ITEM_TYPE_IPV6:
 322                         ipv6 = (struct ipv6_hdr *)&buf[temp_size];
 323                         if (!vlan && !eth)
 324                                 return rte_flow_error_set(error, EINVAL,
 325                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 326                                                 (void *)items->type,
 327                                                 "neither eth nor vlan"
 328                                                 " header found");
 329                         if (vlan && !vlan->eth_proto)
 330                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
 331                         else if (eth && !eth->ether_type)
 332                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
 333                         if (!ipv6->vtc_flow)
 334                                 ipv6->vtc_flow =
 335                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
 336                         if (!ipv6->hop_limits)
 337                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
 338                         break;
 339                 case RTE_FLOW_ITEM_TYPE_UDP:
 340                         udp = (struct udp_hdr *)&buf[temp_size];
 341                         if (!ipv4 && !ipv6)
 342                                 return rte_flow_error_set(error, EINVAL,
 343                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 344                                                 (void *)items->type,
 345                                                 "ip header not found");
 346                         if (ipv4 && !ipv4->next_proto_id)
 347                                 ipv4->next_proto_id = IPPROTO_UDP;
 348                         else if (ipv6 && !ipv6->proto)
 349                                 ipv6->proto = IPPROTO_UDP;
 350                         break;
 351                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 352                         vxlan = (struct vxlan_hdr *)&buf[temp_size];
 353                         if (!udp)
 354                                 return rte_flow_error_set(error, EINVAL,
 355                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 356                                                 (void *)items->type,
 357                                                 "udp header not found");
 358                         if (!udp->dst_port)
 359                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
 360                         if (!vxlan->vx_flags)
 361                                 vxlan->vx_flags =
 362                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
 363                         break;
 364                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 365                         vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
 366                         if (!udp)
 367                                 return rte_flow_error_set(error, EINVAL,
 368                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 369                                                 (void *)items->type,
 370                                                 "udp header not found");
 371                         if (!vxlan_gpe->proto)
 372                                 return rte_flow_error_set(error, EINVAL,
 373                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 374                                                 (void *)items->type,
 375                                                 "next protocol not found");
 376                         if (!udp->dst_port)
 377                                 udp->dst_port =
 378                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
 379                         if (!vxlan_gpe->vx_flags)
 380                                 vxlan_gpe->vx_flags =
 381                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
 382                         break;
 383                 case RTE_FLOW_ITEM_TYPE_GRE:
 384                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 385                         gre = (struct gre_hdr *)&buf[temp_size];
 386                         if (!gre->proto)
 387                                 return rte_flow_error_set(error, EINVAL,
 388                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 389                                                 (void *)items->type,
 390                                                 "next protocol not found");
 391                         if (!ipv4 && !ipv6)
 392                                 return rte_flow_error_set(error, EINVAL,
 393                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 394                                                 (void *)items->type,
 395                                                 "ip header not found");
 396                         if (ipv4 && !ipv4->next_proto_id)
 397                                 ipv4->next_proto_id = IPPROTO_GRE;
 398                         else if (ipv6 && !ipv6->proto)
 399                                 ipv6->proto = IPPROTO_GRE;
 400                         break;
 401                 case RTE_FLOW_ITEM_TYPE_VOID:
 402                         break;
 403                 default:
 404                         return rte_flow_error_set(error, EINVAL,
 405                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 406                                                   (void *)items->type,
 407                                                   "unsupported item type");
 408                         break;
 409                 }
 410                 temp_size += len;
 411         }
 412         *size = temp_size;
 413         return 0;
 414 }
 415
 416 /**
 417  * Convert L2 encap action to DV specification.
 418  *
 419  * @param[in] dev
 420  *   Pointer to rte_eth_dev structure.
 421  * @param[in] action
 422  *   Pointer to action structure.
 423  * @param[out] error
 424  *   Pointer to the error structure.
 425  *
 426  * @return
 427  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 428  */
 429 static struct ibv_flow_action *
 430 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
 431                                const struct rte_flow_action *action,
 432                                struct rte_flow_error *error)
 433 {
 434         struct ibv_flow_action *verbs_action = NULL;
 435         const struct rte_flow_item *encap_data;
 436         struct priv *priv = dev->data->dev_private;
 437         uint8_t buf[MLX5_ENCAP_MAX_LEN];
 438         size_t size = 0;
 439         int convert_result = 0;
 440
 441         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
 442                 encap_data = ((const struct rte_flow_action_vxlan_encap *)
 443                                                 action->conf)->definition;
 444         else
 445                 encap_data = ((const struct rte_flow_action_nvgre_encap *)
 446                                                 action->conf)->definition;
 447         convert_result = flow_dv_convert_encap_data(encap_data, buf,
 448                                                     &size, error);
 449         if (convert_result)
 450                 return NULL;
 451         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 452                 (priv->ctx, size, buf,
 453                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
 454                  MLX5DV_FLOW_TABLE_TYPE_NIC_TX);
 455         if (!verbs_action)
 456                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 457                                    NULL, "cannot create L2 encap action");
 458         return verbs_action;
 459 }
 460
 461 /**
 462  * Convert L2 decap action to DV specification.
 463  *
 464  * @param[in] dev
 465  *   Pointer to rte_eth_dev structure.
 466  * @param[out] error
 467  *   Pointer to the error structure.
 468  *
 469  * @return
 470  *   Pointer to action on success, NULL otherwise and rte_errno is set.
 471  */
 472 static struct ibv_flow_action *
 473 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
 474                                struct rte_flow_error *error)
 475 {
 476         struct ibv_flow_action *verbs_action = NULL;
 477         struct priv *priv = dev->data->dev_private;
 478
 479         verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
 480                 (priv->ctx, 0, NULL,
 481                  MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
 482                  MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
 483         if (!verbs_action)
 484                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
 485                                    NULL, "cannot create L2 decap action");
 486         return verbs_action;
 487 }
 488
 489 /**
 490  * Verify the @p attributes will be correctly understood by the NIC and store
 491  * them in the @p flow if everything is correct.
 492  *
 493  * @param[in] dev
 494  *   Pointer to dev struct.
 495  * @param[in] attributes
 496  *   Pointer to flow attributes
 497  * @param[out] error
 498  *   Pointer to error structure.
 499  *
 500  * @return
 501  *   0 on success, a negative errno value otherwise and rte_errno is set.
 502  */
 503 static int
 504 flow_dv_validate_attributes(struct rte_eth_dev *dev,
 505                             const struct rte_flow_attr *attributes,
 506                             struct rte_flow_error *error)
 507 {
 508         struct priv *priv = dev->data->dev_private;
 509         uint32_t priority_max = priv->config.flow_prio - 1;
 510
 511         if (attributes->group)
 512                 return rte_flow_error_set(error, ENOTSUP,
 513                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 514                                           NULL,
 515                                           "groups is not supported");
 516         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
 517             attributes->priority >= priority_max)
 518                 return rte_flow_error_set(error, ENOTSUP,
 519                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 520                                           NULL,
 521                                           "priority out of range");
 522         if (attributes->transfer)
 523                 return rte_flow_error_set(error, ENOTSUP,
 524                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
 525                                           NULL,
 526                                           "transfer is not supported");
 527         if (!(attributes->egress ^ attributes->ingress))
 528                 return rte_flow_error_set(error, ENOTSUP,
 529                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 530                                           "must specify exactly one of "
 531                                           "ingress or egress");
 532         return 0;
 533 }
 534
 535 /**
 536  * Internal validation function. For validating both actions and items.
 537  *
 538  * @param[in] dev
 539  *   Pointer to the rte_eth_dev structure.
 540  * @param[in] attr
 541  *   Pointer to the flow attributes.
 542  * @param[in] items
 543  *   Pointer to the list of items.
 544  * @param[in] actions
 545  *   Pointer to the list of actions.
 546  * @param[out] error
 547  *   Pointer to the error structure.
 548  *
 549  * @return
 550  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 551  */
 552 static int
 553 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 554                  const struct rte_flow_item items[],
 555                  const struct rte_flow_action actions[],
 556                  struct rte_flow_error *error)
 557 {
 558         int ret;
 559         uint64_t action_flags = 0;
 560         uint64_t item_flags = 0;
 561         int tunnel = 0;
 562         uint8_t next_protocol = 0xff;
 563         int actions_n = 0;
 564
 565         if (items == NULL)
 566                 return -1;
 567         ret = flow_dv_validate_attributes(dev, attr, error);
 568         if (ret < 0)
 569                 return ret;
 570         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 571                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 572                 switch (items->type) {
 573                 case RTE_FLOW_ITEM_TYPE_VOID:
 574                         break;
 575                 case RTE_FLOW_ITEM_TYPE_ETH:
 576                         ret = mlx5_flow_validate_item_eth(items, item_flags,
 577                                                           error);
 578                         if (ret < 0)
 579                                 return ret;
 580                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 581                                                MLX5_FLOW_LAYER_OUTER_L2;
 582                         break;
 583                 case RTE_FLOW_ITEM_TYPE_VLAN:
 584                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
 585                                                            error);
 586                         if (ret < 0)
 587                                 return ret;
 588                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
 589                                                MLX5_FLOW_LAYER_OUTER_VLAN;
 590                         break;
 591                 case RTE_FLOW_ITEM_TYPE_IPV4:
 592                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
 593                                                            error);
 594                         if (ret < 0)
 595                                 return ret;
 596                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 597                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 598                         if (items->mask != NULL &&
 599                             ((const struct rte_flow_item_ipv4 *)
 600                              items->mask)->hdr.next_proto_id)
 601                                 next_protocol =
 602                                         ((const struct rte_flow_item_ipv4 *)
 603                                          (items->spec))->hdr.next_proto_id;
 604                         break;
 605                 case RTE_FLOW_ITEM_TYPE_IPV6:
 606                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
 607                                                            error);
 608                         if (ret < 0)
 609                                 return ret;
 610                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 611                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 612                         if (items->mask != NULL &&
 613                             ((const struct rte_flow_item_ipv6 *)
 614                              items->mask)->hdr.proto)
 615                                 next_protocol =
 616                                         ((const struct rte_flow_item_ipv6 *)
 617                                          items->spec)->hdr.proto;
 618                         break;
 619                 case RTE_FLOW_ITEM_TYPE_TCP:
 620                         ret = mlx5_flow_validate_item_tcp
 621                                                 (items, item_flags,
 622                                                  next_protocol,
 623                                                  &rte_flow_item_tcp_mask,
 624                                                  error);
 625                         if (ret < 0)
 626                                 return ret;
 627                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 628                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
 629                         break;
 630                 case RTE_FLOW_ITEM_TYPE_UDP:
 631                         ret = mlx5_flow_validate_item_udp(items, item_flags,
 632                                                           next_protocol,
 633                                                           error);
 634                         if (ret < 0)
 635                                 return ret;
 636                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 637                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
 638                         break;
 639                 case RTE_FLOW_ITEM_TYPE_GRE:
 640                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 641                         ret = mlx5_flow_validate_item_gre(items, item_flags,
 642                                                           next_protocol, error);
 643                         if (ret < 0)
 644                                 return ret;
 645                         item_flags |= MLX5_FLOW_LAYER_GRE;
 646                         break;
 647                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 648                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
 649                                                             error);
 650                         if (ret < 0)
 651                                 return ret;
 652                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
 653                         break;
 654                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 655                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
 656                                                                 item_flags, dev,
 657                                                                 error);
 658                         if (ret < 0)
 659                                 return ret;
 660                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 661                         break;
 662                 case RTE_FLOW_ITEM_TYPE_META:
 663                         ret = flow_dv_validate_item_meta(dev, items, attr,
 664                                                          error);
 665                         if (ret < 0)
 666                                 return ret;
 667                         item_flags |= MLX5_FLOW_ITEM_METADATA;
 668                         break;
 669                 default:
 670                         return rte_flow_error_set(error, ENOTSUP,
 671                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 672                                                   NULL, "item not supported");
 673                 }
 674         }
 675         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 676                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
 677                         return rte_flow_error_set(error, ENOTSUP,
 678                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 679                                                   actions, "too many actions");
 680                 switch (actions->type) {
 681                 case RTE_FLOW_ACTION_TYPE_VOID:
 682                         break;
 683                 case RTE_FLOW_ACTION_TYPE_FLAG:
 684                         ret = mlx5_flow_validate_action_flag(action_flags,
 685                                                              attr, error);
 686                         if (ret < 0)
 687                                 return ret;
 688                         action_flags |= MLX5_FLOW_ACTION_FLAG;
 689                         ++actions_n;
 690                         break;
 691                 case RTE_FLOW_ACTION_TYPE_MARK:
 692                         ret = mlx5_flow_validate_action_mark(actions,
 693                                                              action_flags,
 694                                                              attr, error);
 695                         if (ret < 0)
 696                                 return ret;
 697                         action_flags |= MLX5_FLOW_ACTION_MARK;
 698                         ++actions_n;
 699                         break;
 700                 case RTE_FLOW_ACTION_TYPE_DROP:
 701                         ret = mlx5_flow_validate_action_drop(action_flags,
 702                                                              attr, error);
 703                         if (ret < 0)
 704                                 return ret;
 705                         action_flags |= MLX5_FLOW_ACTION_DROP;
 706                         ++actions_n;
 707                         break;
 708                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 709                         ret = mlx5_flow_validate_action_queue(actions,
 710                                                               action_flags, dev,
 711                                                               attr, error);
 712                         if (ret < 0)
 713                                 return ret;
 714                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
 715                         ++actions_n;
 716                         break;
 717                 case RTE_FLOW_ACTION_TYPE_RSS:
 718                         ret = mlx5_flow_validate_action_rss(actions,
 719                                                             action_flags, dev,
 720                                                             attr, error);
 721                         if (ret < 0)
 722                                 return ret;
 723                         action_flags |= MLX5_FLOW_ACTION_RSS;
 724                         ++actions_n;
 725                         break;
 726                 case RTE_FLOW_ACTION_TYPE_COUNT:
 727                         ret = mlx5_flow_validate_action_count(dev, attr, error);
 728                         if (ret < 0)
 729                                 return ret;
 730                         action_flags |= MLX5_FLOW_ACTION_COUNT;
 731                         ++actions_n;
 732                         break;
 733                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
 734                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
 735                         ret = flow_dv_validate_action_l2_encap(action_flags,
 736                                                                actions, attr,
 737                                                                error);
 738                         if (ret < 0)
 739                                 return ret;
 740                         action_flags |= actions->type ==
 741                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
 742                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
 743                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
 744
 745                         ++actions_n;
 746                         break;
 747                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
 748                         ret = flow_dv_validate_action_l2_decap(action_flags,
 749                                                                attr, error);
 750                         if (ret < 0)
 751                                 return ret;
 752                         action_flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
 753                         ++actions_n;
 754                         break;
 755                 default:
 756                         return rte_flow_error_set(error, ENOTSUP,
 757                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 758                                                   actions,
 759                                                   "action not supported");
 760                 }
 761         }
 762         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
 763                 return rte_flow_error_set(error, EINVAL,
 764                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
 765                                           "no fate action is found");
 766         return 0;
 767 }
 768
 769 /**
 770  * Internal preparation function. Allocates the DV flow size,
 771  * this size is constant.
 772  *
 773  * @param[in] attr
 774  *   Pointer to the flow attributes.
 775  * @param[in] items
 776  *   Pointer to the list of items.
 777  * @param[in] actions
 778  *   Pointer to the list of actions.
 779  * @param[out] item_flags
 780  *   Pointer to bit mask of all items detected.
 781  * @param[out] action_flags
 782  *   Pointer to bit mask of all actions detected.
 783  * @param[out] error
 784  *   Pointer to the error structure.
 785  *
 786  * @return
 787  *   Pointer to mlx5_flow object on success,
 788  *   otherwise NULL and rte_ernno is set.
 789  */
 790 static struct mlx5_flow *
 791 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
 792                 const struct rte_flow_item items[] __rte_unused,
 793                 const struct rte_flow_action actions[] __rte_unused,
 794                 uint64_t *item_flags __rte_unused,
 795                 uint64_t *action_flags __rte_unused,
 796                 struct rte_flow_error *error)
 797 {
 798         uint32_t size = sizeof(struct mlx5_flow);
 799         struct mlx5_flow *flow;
 800
 801         flow = rte_calloc(__func__, 1, size, 0);
 802         if (!flow) {
 803                 rte_flow_error_set(error, ENOMEM,
 804                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 805                                    "not enough memory to create flow");
 806                 return NULL;
 807         }
 808         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
 809         return flow;
 810 }
 811
 812 /**
 813  * Add Ethernet item to matcher and to the value.
 814  *
 815  * @param[in, out] matcher
 816  *   Flow matcher.
 817  * @param[in, out] key
 818  *   Flow matcher value.
 819  * @param[in] item
 820  *   Flow pattern to translate.
 821  * @param[in] inner
 822  *   Item is inner pattern.
 823  */
 824 static void
 825 flow_dv_translate_item_eth(void *matcher, void *key,
 826                            const struct rte_flow_item *item, int inner)
 827 {
 828         const struct rte_flow_item_eth *eth_m = item->mask;
 829         const struct rte_flow_item_eth *eth_v = item->spec;
 830         const struct rte_flow_item_eth nic_mask = {
 831                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 832                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 833                 .type = RTE_BE16(0xffff),
 834         };
 835         void *headers_m;
 836         void *headers_v;
 837         char *l24_v;
 838         unsigned int i;
 839
 840         if (!eth_v)
 841                 return;
 842         if (!eth_m)
 843                 eth_m = &nic_mask;
 844         if (inner) {
 845                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 846                                          inner_headers);
 847                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 848         } else {
 849                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 850                                          outer_headers);
 851                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 852         }
 853         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
 854                &eth_m->dst, sizeof(eth_m->dst));
 855         /* The value must be in the range of the mask. */
 856         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
 857         for (i = 0; i < sizeof(eth_m->dst); ++i)
 858                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
 859         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
 860                &eth_m->src, sizeof(eth_m->src));
 861         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
 862         /* The value must be in the range of the mask. */
 863         for (i = 0; i < sizeof(eth_m->dst); ++i)
 864                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
 865         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
 866                  rte_be_to_cpu_16(eth_m->type));
 867         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
 868         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
 869 }
 870
 871 /**
 872  * Add VLAN item to matcher and to the value.
 873  *
 874  * @param[in, out] matcher
 875  *   Flow matcher.
 876  * @param[in, out] key
 877  *   Flow matcher value.
 878  * @param[in] item
 879  *   Flow pattern to translate.
 880  * @param[in] inner
 881  *   Item is inner pattern.
 882  */
 883 static void
 884 flow_dv_translate_item_vlan(void *matcher, void *key,
 885                             const struct rte_flow_item *item,
 886                             int inner)
 887 {
 888         const struct rte_flow_item_vlan *vlan_m = item->mask;
 889         const struct rte_flow_item_vlan *vlan_v = item->spec;
 890         const struct rte_flow_item_vlan nic_mask = {
 891                 .tci = RTE_BE16(0x0fff),
 892                 .inner_type = RTE_BE16(0xffff),
 893         };
 894         void *headers_m;
 895         void *headers_v;
 896         uint16_t tci_m;
 897         uint16_t tci_v;
 898
 899         if (!vlan_v)
 900                 return;
 901         if (!vlan_m)
 902                 vlan_m = &nic_mask;
 903         if (inner) {
 904                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 905                                          inner_headers);
 906                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 907         } else {
 908                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 909                                          outer_headers);
 910                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 911         }
 912         tci_m = rte_be_to_cpu_16(vlan_m->tci);
 913         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
 914         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
 915         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
 916         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
 917         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
 918         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
 919         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
 920         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
 921         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
 922 }
 923
 924 /**
 925  * Add IPV4 item to matcher and to the value.
 926  *
 927  * @param[in, out] matcher
 928  *   Flow matcher.
 929  * @param[in, out] key
 930  *   Flow matcher value.
 931  * @param[in] item
 932  *   Flow pattern to translate.
 933  * @param[in] inner
 934  *   Item is inner pattern.
 935  */
 936 static void
 937 flow_dv_translate_item_ipv4(void *matcher, void *key,
 938                             const struct rte_flow_item *item,
 939                             int inner)
 940 {
 941         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
 942         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
 943         const struct rte_flow_item_ipv4 nic_mask = {
 944                 .hdr = {
 945                         .src_addr = RTE_BE32(0xffffffff),
 946                         .dst_addr = RTE_BE32(0xffffffff),
 947                         .type_of_service = 0xff,
 948                         .next_proto_id = 0xff,
 949                 },
 950         };
 951         void *headers_m;
 952         void *headers_v;
 953         char *l24_m;
 954         char *l24_v;
 955         uint8_t tos;
 956
 957         if (inner) {
 958                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 959                                          inner_headers);
 960                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 961         } else {
 962                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 963                                          outer_headers);
 964                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 965         }
 966         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 967         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
 968         if (!ipv4_v)
 969                 return;
 970         if (!ipv4_m)
 971                 ipv4_m = &nic_mask;
 972         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 973                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 974         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 975                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 976         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
 977         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
 978         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 979                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 980         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 981                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 982         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
 983         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
 984         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
 985         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
 986                  ipv4_m->hdr.type_of_service);
 987         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
 988         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
 989                  ipv4_m->hdr.type_of_service >> 2);
 990         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
 991         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 992                  ipv4_m->hdr.next_proto_id);
 993         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 994                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
 995 }
 996
 997 /**
 998  * Add IPV6 item to matcher and to the value.
 999  *
1000  * @param[in, out] matcher
1001  *   Flow matcher.
1002  * @param[in, out] key
1003  *   Flow matcher value.
1004  * @param[in] item
1005  *   Flow pattern to translate.
1006  * @param[in] inner
1007  *   Item is inner pattern.
1008  */
1009 static void
1010 flow_dv_translate_item_ipv6(void *matcher, void *key,
1011                             const struct rte_flow_item *item,
1012                             int inner)
1013 {
1014         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
1015         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
1016         const struct rte_flow_item_ipv6 nic_mask = {
1017                 .hdr = {
1018                         .src_addr =
1019                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1020                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1021                         .dst_addr =
1022                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
1023                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
1024                         .vtc_flow = RTE_BE32(0xffffffff),
1025                         .proto = 0xff,
1026                         .hop_limits = 0xff,
1027                 },
1028         };
1029         void *headers_m;
1030         void *headers_v;
1031         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1032         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1033         char *l24_m;
1034         char *l24_v;
1035         uint32_t vtc_m;
1036         uint32_t vtc_v;
1037         int i;
1038         int size;
1039
1040         if (inner) {
1041                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1042                                          inner_headers);
1043                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1044         } else {
1045                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1046                                          outer_headers);
1047                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1048         }
1049         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
1050         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
1051         if (!ipv6_v)
1052                 return;
1053         if (!ipv6_m)
1054                 ipv6_m = &nic_mask;
1055         size = sizeof(ipv6_m->hdr.dst_addr);
1056         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1057                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1058         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1059                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1060         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
1061         for (i = 0; i < size; ++i)
1062                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
1063         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1064                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1065         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1066                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
1067         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
1068         for (i = 0; i < size; ++i)
1069                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
1070         /* TOS. */
1071         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
1072         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
1073         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
1074         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
1075         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
1076         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
1077         /* Label. */
1078         if (inner) {
1079                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
1080                          vtc_m);
1081                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
1082                          vtc_v);
1083         } else {
1084                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
1085                          vtc_m);
1086                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
1087                          vtc_v);
1088         }
1089         /* Protocol. */
1090         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
1091                  ipv6_m->hdr.proto);
1092         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1093                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
1094 }
1095
1096 /**
1097  * Add TCP item to matcher and to the value.
1098  *
1099  * @param[in, out] matcher
1100  *   Flow matcher.
1101  * @param[in, out] key
1102  *   Flow matcher value.
1103  * @param[in] item
1104  *   Flow pattern to translate.
1105  * @param[in] inner
1106  *   Item is inner pattern.
1107  */
1108 static void
1109 flow_dv_translate_item_tcp(void *matcher, void *key,
1110                            const struct rte_flow_item *item,
1111                            int inner)
1112 {
1113         const struct rte_flow_item_tcp *tcp_m = item->mask;
1114         const struct rte_flow_item_tcp *tcp_v = item->spec;
1115         void *headers_m;
1116         void *headers_v;
1117
1118         if (inner) {
1119                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1120                                          inner_headers);
1121                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1122         } else {
1123                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1124                                          outer_headers);
1125                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1126         }
1127         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1128         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
1129         if (!tcp_v)
1130                 return;
1131         if (!tcp_m)
1132                 tcp_m = &rte_flow_item_tcp_mask;
1133         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
1134                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
1135         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
1136                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
1137         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
1138                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
1139         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
1140                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
1141 }
1142
1143 /**
1144  * Add UDP item to matcher and to the value.
1145  *
1146  * @param[in, out] matcher
1147  *   Flow matcher.
1148  * @param[in, out] key
1149  *   Flow matcher value.
1150  * @param[in] item
1151  *   Flow pattern to translate.
1152  * @param[in] inner
1153  *   Item is inner pattern.
1154  */
1155 static void
1156 flow_dv_translate_item_udp(void *matcher, void *key,
1157                            const struct rte_flow_item *item,
1158                            int inner)
1159 {
1160         const struct rte_flow_item_udp *udp_m = item->mask;
1161         const struct rte_flow_item_udp *udp_v = item->spec;
1162         void *headers_m;
1163         void *headers_v;
1164
1165         if (inner) {
1166                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1167                                          inner_headers);
1168                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1169         } else {
1170                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1171                                          outer_headers);
1172                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1173         }
1174         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1175         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
1176         if (!udp_v)
1177                 return;
1178         if (!udp_m)
1179                 udp_m = &rte_flow_item_udp_mask;
1180         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
1181                  rte_be_to_cpu_16(udp_m->hdr.src_port));
1182         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
1183                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
1184         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
1185                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
1186         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
1187                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
1188 }
1189
1190 /**
1191  * Add GRE item to matcher and to the value.
1192  *
1193  * @param[in, out] matcher
1194  *   Flow matcher.
1195  * @param[in, out] key
1196  *   Flow matcher value.
1197  * @param[in] item
1198  *   Flow pattern to translate.
1199  * @param[in] inner
1200  *   Item is inner pattern.
1201  */
1202 static void
1203 flow_dv_translate_item_gre(void *matcher, void *key,
1204                            const struct rte_flow_item *item,
1205                            int inner)
1206 {
1207         const struct rte_flow_item_gre *gre_m = item->mask;
1208         const struct rte_flow_item_gre *gre_v = item->spec;
1209         void *headers_m;
1210         void *headers_v;
1211         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1212         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1213
1214         if (inner) {
1215                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1216                                          inner_headers);
1217                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1218         } else {
1219                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1220                                          outer_headers);
1221                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1222         }
1223         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1224         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
1225         if (!gre_v)
1226                 return;
1227         if (!gre_m)
1228                 gre_m = &rte_flow_item_gre_mask;
1229         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
1230                  rte_be_to_cpu_16(gre_m->protocol));
1231         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
1232                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
1233 }
1234
1235 /**
1236  * Add NVGRE item to matcher and to the value.
1237  *
1238  * @param[in, out] matcher
1239  *   Flow matcher.
1240  * @param[in, out] key
1241  *   Flow matcher value.
1242  * @param[in] item
1243  *   Flow pattern to translate.
1244  * @param[in] inner
1245  *   Item is inner pattern.
1246  */
1247 static void
1248 flow_dv_translate_item_nvgre(void *matcher, void *key,
1249                              const struct rte_flow_item *item,
1250                              int inner)
1251 {
1252         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
1253         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
1254         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1255         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1256         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
1257         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
1258         char *gre_key_m;
1259         char *gre_key_v;
1260         int size;
1261         int i;
1262
1263         flow_dv_translate_item_gre(matcher, key, item, inner);
1264         if (!nvgre_v)
1265                 return;
1266         if (!nvgre_m)
1267                 nvgre_m = &rte_flow_item_nvgre_mask;
1268         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
1269         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
1270         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
1271         memcpy(gre_key_m, tni_flow_id_m, size);
1272         for (i = 0; i < size; ++i)
1273                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
1274 }
1275
1276 /**
1277  * Add VXLAN item to matcher and to the value.
1278  *
1279  * @param[in, out] matcher
1280  *   Flow matcher.
1281  * @param[in, out] key
1282  *   Flow matcher value.
1283  * @param[in] item
1284  *   Flow pattern to translate.
1285  * @param[in] inner
1286  *   Item is inner pattern.
1287  */
1288 static void
1289 flow_dv_translate_item_vxlan(void *matcher, void *key,
1290                              const struct rte_flow_item *item,
1291                              int inner)
1292 {
1293         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
1294         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
1295         void *headers_m;
1296         void *headers_v;
1297         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1298         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1299         char *vni_m;
1300         char *vni_v;
1301         uint16_t dport;
1302         int size;
1303         int i;
1304
1305         if (inner) {
1306                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1307                                          inner_headers);
1308                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1309         } else {
1310                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1311                                          outer_headers);
1312                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1313         }
1314         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
1315                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
1316         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
1317                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
1318                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
1319         }
1320         if (!vxlan_v)
1321                 return;
1322         if (!vxlan_m)
1323                 vxlan_m = &rte_flow_item_vxlan_mask;
1324         size = sizeof(vxlan_m->vni);
1325         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
1326         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
1327         memcpy(vni_m, vxlan_m->vni, size);
1328         for (i = 0; i < size; ++i)
1329                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
1330 }
1331
1332 /**
1333  * Add META item to matcher
1334  *
1335  * @param[in, out] matcher
1336  *   Flow matcher.
1337  * @param[in, out] key
1338  *   Flow matcher value.
1339  * @param[in] item
1340  *   Flow pattern to translate.
1341  * @param[in] inner
1342  *   Item is inner pattern.
1343  */
1344 static void
1345 flow_dv_translate_item_meta(void *matcher, void *key,
1346                             const struct rte_flow_item *item)
1347 {
1348         const struct rte_flow_item_meta *meta_m;
1349         const struct rte_flow_item_meta *meta_v;
1350         void *misc2_m =
1351                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
1352         void *misc2_v =
1353                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
1354
1355         meta_m = (const void *)item->mask;
1356         if (!meta_m)
1357                 meta_m = &rte_flow_item_meta_mask;
1358         meta_v = (const void *)item->spec;
1359         if (meta_v) {
1360                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
1361                          rte_be_to_cpu_32(meta_m->data));
1362                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
1363                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
1364         }
1365 }
1366
1367 /**
1368  * Update the matcher and the value based the selected item.
1369  *
1370  * @param[in, out] matcher
1371  *   Flow matcher.
1372  * @param[in, out] key
1373  *   Flow matcher value.
1374  * @param[in] item
1375  *   Flow pattern to translate.
1376  * @param[in, out] dev_flow
1377  *   Pointer to the mlx5_flow.
1378  * @param[in] inner
1379  *   Item is inner pattern.
1380  */
1381 static void
1382 flow_dv_create_item(void *matcher, void *key,
1383                     const struct rte_flow_item *item,
1384                     struct mlx5_flow *dev_flow,
1385                     int inner)
1386 {
1387         struct mlx5_flow_dv_matcher *tmatcher = matcher;
1388
1389         switch (item->type) {
1390         case RTE_FLOW_ITEM_TYPE_ETH:
1391                 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
1392                                            inner);
1393                 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
1394                 break;
1395         case RTE_FLOW_ITEM_TYPE_VLAN:
1396                 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
1397                                             inner);
1398                 break;
1399         case RTE_FLOW_ITEM_TYPE_IPV4:
1400                 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
1401                                             inner);
1402                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1403                 dev_flow->dv.hash_fields |=
1404                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1405                                                     MLX5_IPV4_LAYER_TYPES,
1406                                                     MLX5_IPV4_IBV_RX_HASH);
1407                 break;
1408         case RTE_FLOW_ITEM_TYPE_IPV6:
1409                 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
1410                                             inner);
1411                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1412                 dev_flow->dv.hash_fields |=
1413                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1414                                                     MLX5_IPV6_LAYER_TYPES,
1415                                                     MLX5_IPV6_IBV_RX_HASH);
1416                 break;
1417         case RTE_FLOW_ITEM_TYPE_TCP:
1418                 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
1419                                            inner);
1420                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1421                 dev_flow->dv.hash_fields |=
1422                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1423                                                     ETH_RSS_TCP,
1424                                                     (IBV_RX_HASH_SRC_PORT_TCP |
1425                                                      IBV_RX_HASH_DST_PORT_TCP));
1426                 break;
1427         case RTE_FLOW_ITEM_TYPE_UDP:
1428                 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
1429                                            inner);
1430                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1431                 dev_flow->verbs.hash_fields |=
1432                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1433                                                     ETH_RSS_UDP,
1434                                                     (IBV_RX_HASH_SRC_PORT_UDP |
1435                                                      IBV_RX_HASH_DST_PORT_UDP));
1436                 break;
1437         case RTE_FLOW_ITEM_TYPE_GRE:
1438                 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
1439                                            inner);
1440                 break;
1441         case RTE_FLOW_ITEM_TYPE_NVGRE:
1442                 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
1443                                              inner);
1444                 break;
1445         case RTE_FLOW_ITEM_TYPE_VXLAN:
1446         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1447                 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
1448                                              inner);
1449                 break;
1450         case RTE_FLOW_ITEM_TYPE_META:
1451                 flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
1452                 break;
1453         default:
1454                 break;
1455         }
1456 }
1457
1458 /**
1459  * Store the requested actions in an array.
1460  *
1461  * @param[in] dev
1462  *   Pointer to rte_eth_dev structure.
1463  * @param[in] action
1464  *   Flow action to translate.
1465  * @param[in, out] dev_flow
1466  *   Pointer to the mlx5_flow.
1467  * @param[out] error
1468  *   Pointer to the error structure.
1469  *
1470  * @return
1471  *   0 on success, a negative errno value otherwise and rte_errno is set.
1472  */
1473 static int
1474 flow_dv_create_action(struct rte_eth_dev *dev,
1475                       const struct rte_flow_action *action,
1476                       struct mlx5_flow *dev_flow,
1477                       struct rte_flow_error *error)
1478 {
1479         const struct rte_flow_action_queue *queue;
1480         const struct rte_flow_action_rss *rss;
1481         int actions_n = dev_flow->dv.actions_n;
1482         struct rte_flow *flow = dev_flow->flow;
1483
1484         switch (action->type) {
1485         case RTE_FLOW_ACTION_TYPE_VOID:
1486                 break;
1487         case RTE_FLOW_ACTION_TYPE_FLAG:
1488                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1489                 dev_flow->dv.actions[actions_n].tag_value =
1490                         mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
1491                 actions_n++;
1492                 flow->actions |= MLX5_FLOW_ACTION_FLAG;
1493                 break;
1494         case RTE_FLOW_ACTION_TYPE_MARK:
1495                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1496                 dev_flow->dv.actions[actions_n].tag_value =
1497                         mlx5_flow_mark_set
1498                         (((const struct rte_flow_action_mark *)
1499                           (action->conf))->id);
1500                 flow->actions |= MLX5_FLOW_ACTION_MARK;
1501                 actions_n++;
1502                 break;
1503         case RTE_FLOW_ACTION_TYPE_DROP:
1504                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
1505                 flow->actions |= MLX5_FLOW_ACTION_DROP;
1506                 break;
1507         case RTE_FLOW_ACTION_TYPE_QUEUE:
1508                 queue = action->conf;
1509                 flow->rss.queue_num = 1;
1510                 (*flow->queue)[0] = queue->index;
1511                 flow->actions |= MLX5_FLOW_ACTION_QUEUE;
1512                 break;
1513         case RTE_FLOW_ACTION_TYPE_RSS:
1514                 rss = action->conf;
1515                 if (flow->queue)
1516                         memcpy((*flow->queue), rss->queue,
1517                                rss->queue_num * sizeof(uint16_t));
1518                 flow->rss.queue_num = rss->queue_num;
1519                 memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
1520                 flow->rss.types = rss->types;
1521                 flow->rss.level = rss->level;
1522                 /* Added to array only in apply since we need the QP */
1523                 flow->actions |= MLX5_FLOW_ACTION_RSS;
1524                 break;
1525         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1526         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
1527                 dev_flow->dv.actions[actions_n].type =
1528                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1529                 dev_flow->dv.actions[actions_n].action =
1530                                 flow_dv_create_action_l2_encap(dev, action,
1531                                                                error);
1532                 if (!(dev_flow->dv.actions[actions_n].action))
1533                         return -rte_errno;
1534                 dev_flow->dv.encap_decap_verbs_action =
1535                         dev_flow->dv.actions[actions_n].action;
1536                 flow->actions |= action->type ==
1537                                  RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
1538                                  MLX5_FLOW_ACTION_VXLAN_ENCAP :
1539                                  MLX5_FLOW_ACTION_NVGRE_ENCAP;
1540                 actions_n++;
1541                 break;
1542         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1543                 dev_flow->dv.actions[actions_n].type =
1544                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1545                 dev_flow->dv.actions[actions_n].action =
1546                                 flow_dv_create_action_l2_decap(dev, error);
1547                 if (!(dev_flow->dv.actions[actions_n].action))
1548                         return -rte_errno;
1549                 dev_flow->dv.encap_decap_verbs_action =
1550                         dev_flow->dv.actions[actions_n].action;
1551                 flow->actions |= MLX5_FLOW_ACTION_VXLAN_DECAP;
1552                 actions_n++;
1553                 break;
1554         default:
1555                 break;
1556         }
1557         dev_flow->dv.actions_n = actions_n;
1558         return 0;
1559 }
1560
1561 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
1562
1563 #define HEADER_IS_ZERO(match_criteria, headers)                              \
1564         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
1565                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1566
1567 /**
1568  * Calculate flow matcher enable bitmap.
1569  *
1570  * @param match_criteria
1571  *   Pointer to flow matcher criteria.
1572  *
1573  * @return
1574  *   Bitmap of enabled fields.
1575  */
1576 static uint8_t
1577 flow_dv_matcher_enable(uint32_t *match_criteria)
1578 {
1579         uint8_t match_criteria_enable;
1580
1581         match_criteria_enable =
1582                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1583                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
1584         match_criteria_enable |=
1585                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1586                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
1587         match_criteria_enable |=
1588                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1589                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
1590         match_criteria_enable |=
1591                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
1592                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
1593
1594         return match_criteria_enable;
1595 }
1596
1597 /**
1598  * Register the flow matcher.
1599  *
1600  * @param dev[in, out]
1601  *   Pointer to rte_eth_dev structure.
1602  * @param[in, out] matcher
1603  *   Pointer to flow matcher.
1604  * @parm[in, out] dev_flow
1605  *   Pointer to the dev_flow.
1606  * @param[out] error
1607  *   pointer to error structure.
1608  *
1609  * @return
1610  *   0 on success otherwise -errno and errno is set.
1611  */
1612 static int
1613 flow_dv_matcher_register(struct rte_eth_dev *dev,
1614                          struct mlx5_flow_dv_matcher *matcher,
1615                          struct mlx5_flow *dev_flow,
1616                          struct rte_flow_error *error)
1617 {
1618         struct priv *priv = dev->data->dev_private;
1619         struct mlx5_flow_dv_matcher *cache_matcher;
1620         struct mlx5dv_flow_matcher_attr dv_attr = {
1621                 .type = IBV_FLOW_ATTR_NORMAL,
1622                 .match_mask = (void *)&matcher->mask,
1623         };
1624
1625         /* Lookup from cache. */
1626         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1627                 if (matcher->crc == cache_matcher->crc &&
1628                     matcher->priority == cache_matcher->priority &&
1629                     matcher->egress == cache_matcher->egress &&
1630                     !memcmp((const void *)matcher->mask.buf,
1631                             (const void *)cache_matcher->mask.buf,
1632                             cache_matcher->mask.size)) {
1633                         DRV_LOG(DEBUG,
1634                                 "priority %hd use %s matcher %p: refcnt %d++",
1635                                 cache_matcher->priority,
1636                                 cache_matcher->egress ? "tx" : "rx",
1637                                 (void *)cache_matcher,
1638                                 rte_atomic32_read(&cache_matcher->refcnt));
1639                         rte_atomic32_inc(&cache_matcher->refcnt);
1640                         dev_flow->dv.matcher = cache_matcher;
1641                         return 0;
1642                 }
1643         }
1644         /* Register new matcher. */
1645         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1646         if (!cache_matcher)
1647                 return rte_flow_error_set(error, ENOMEM,
1648                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1649                                           "cannot allocate matcher memory");
1650         *cache_matcher = *matcher;
1651         dv_attr.match_criteria_enable =
1652                 flow_dv_matcher_enable(cache_matcher->mask.buf);
1653         dv_attr.priority = matcher->priority;
1654         if (matcher->egress)
1655                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1656         cache_matcher->matcher_object =
1657                 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
1658         if (!cache_matcher->matcher_object) {
1659                 rte_free(cache_matcher);
1660                 return rte_flow_error_set(error, ENOMEM,
1661                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1662                                           NULL, "cannot create matcher");
1663         }
1664         rte_atomic32_inc(&cache_matcher->refcnt);
1665         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1666         dev_flow->dv.matcher = cache_matcher;
1667         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1668                 cache_matcher->priority,
1669                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1670                 rte_atomic32_read(&cache_matcher->refcnt));
1671         return 0;
1672 }
1673
1674 /**
1675  * Fill the flow with DV spec.
1676  *
1677  * @param[in] dev
1678  *   Pointer to rte_eth_dev structure.
1679  * @param[in, out] dev_flow
1680  *   Pointer to the sub flow.
1681  * @param[in] attr
1682  *   Pointer to the flow attributes.
1683  * @param[in] items
1684  *   Pointer to the list of items.
1685  * @param[in] actions
1686  *   Pointer to the list of actions.
1687  * @param[out] error
1688  *   Pointer to the error structure.
1689  *
1690  * @return
1691  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1692  */
1693 static int
1694 flow_dv_translate(struct rte_eth_dev *dev,
1695                   struct mlx5_flow *dev_flow,
1696                   const struct rte_flow_attr *attr,
1697                   const struct rte_flow_item items[],
1698                   const struct rte_flow_action actions[] __rte_unused,
1699                   struct rte_flow_error *error)
1700 {
1701         struct priv *priv = dev->data->dev_private;
1702         uint64_t priority = attr->priority;
1703         struct mlx5_flow_dv_matcher matcher = {
1704                 .mask = {
1705                         .size = sizeof(matcher.mask.buf),
1706                 },
1707         };
1708         void *match_value = dev_flow->dv.value.buf;
1709         int tunnel = 0;
1710
1711         if (priority == MLX5_FLOW_PRIO_RSVD)
1712                 priority = priv->config.flow_prio - 1;
1713         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1714                 tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
1715                 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1716                                     tunnel);
1717         }
1718         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1719                                      matcher.mask.size);
1720         if (priority == MLX5_FLOW_PRIO_RSVD)
1721                 priority = priv->config.flow_prio - 1;
1722         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1723                                                      matcher.priority);
1724         matcher.egress = attr->egress;
1725         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1726                 return -rte_errno;
1727         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1728                 if (flow_dv_create_action(dev, actions, dev_flow, error))
1729                         return -rte_errno;
1730         return 0;
1731 }
1732
1733 /**
1734  * Apply the flow to the NIC.
1735  *
1736  * @param[in] dev
1737  *   Pointer to the Ethernet device structure.
1738  * @param[in, out] flow
1739  *   Pointer to flow structure.
1740  * @param[out] error
1741  *   Pointer to error structure.
1742  *
1743  * @return
1744  *   0 on success, a negative errno value otherwise and rte_errno is set.
1745  */
1746 static int
1747 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1748               struct rte_flow_error *error)
1749 {
1750         struct mlx5_flow_dv *dv;
1751         struct mlx5_flow *dev_flow;
1752         int n;
1753         int err;
1754
1755         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1756                 dv = &dev_flow->dv;
1757                 n = dv->actions_n;
1758                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1759                         dv->hrxq = mlx5_hrxq_drop_new(dev);
1760                         if (!dv->hrxq) {
1761                                 rte_flow_error_set
1762                                         (error, errno,
1763                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1764                                          "cannot get drop hash queue");
1765                                 goto error;
1766                         }
1767                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1768                         dv->actions[n].qp = dv->hrxq->qp;
1769                         n++;
1770                 } else if (flow->actions &
1771                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
1772                         struct mlx5_hrxq *hrxq;
1773                         hrxq = mlx5_hrxq_get(dev, flow->key,
1774                                              MLX5_RSS_HASH_KEY_LEN,
1775                                              dv->hash_fields,
1776                                              (*flow->queue),
1777                                              flow->rss.queue_num);
1778                         if (!hrxq)
1779                                 hrxq = mlx5_hrxq_new
1780                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
1781                                          dv->hash_fields, (*flow->queue),
1782                                          flow->rss.queue_num,
1783                                          !!(dev_flow->layers &
1784                                             MLX5_FLOW_LAYER_TUNNEL));
1785                         if (!hrxq) {
1786                                 rte_flow_error_set
1787                                         (error, rte_errno,
1788                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1789                                          "cannot get hash queue");
1790                                 goto error;
1791                         }
1792                         dv->hrxq = hrxq;
1793                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1794                         dv->actions[n].qp = hrxq->qp;
1795                         n++;
1796                 }
1797                 dv->flow =
1798                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
1799                                                   (void *)&dv->value, n,
1800                                                   dv->actions);
1801                 if (!dv->flow) {
1802                         rte_flow_error_set(error, errno,
1803                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1804                                            NULL,
1805                                            "hardware refuses to create flow");
1806                         goto error;
1807                 }
1808         }
1809         return 0;
1810 error:
1811         err = rte_errno; /* Save rte_errno before cleanup. */
1812         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1813                 struct mlx5_flow_dv *dv = &dev_flow->dv;
1814                 if (dv->hrxq) {
1815                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1816                                 mlx5_hrxq_drop_release(dev);
1817                         else
1818                                 mlx5_hrxq_release(dev, dv->hrxq);
1819                         dv->hrxq = NULL;
1820                 }
1821         }
1822         rte_errno = err; /* Restore rte_errno. */
1823         return -rte_errno;
1824 }
1825
1826 /**
1827  * Release the flow matcher.
1828  *
1829  * @param dev
1830  *   Pointer to Ethernet device.
1831  * @param flow
1832  *   Pointer to mlx5_flow.
1833  *
1834  * @return
1835  *   1 while a reference on it exists, 0 when freed.
1836  */
1837 static int
1838 flow_dv_matcher_release(struct rte_eth_dev *dev,
1839                         struct mlx5_flow *flow)
1840 {
1841         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
1842
1843         assert(matcher->matcher_object);
1844         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
1845                 dev->data->port_id, (void *)matcher,
1846                 rte_atomic32_read(&matcher->refcnt));
1847         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
1848                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
1849                            (matcher->matcher_object));
1850                 LIST_REMOVE(matcher, next);
1851                 rte_free(matcher);
1852                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
1853                         dev->data->port_id, (void *)matcher);
1854                 return 0;
1855         }
1856         return 1;
1857 }
1858
1859 /**
1860  * Remove the flow from the NIC but keeps it in memory.
1861  *
1862  * @param[in] dev
1863  *   Pointer to Ethernet device.
1864  * @param[in, out] flow
1865  *   Pointer to flow structure.
1866  */
1867 static void
1868 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1869 {
1870         struct mlx5_flow_dv *dv;
1871         struct mlx5_flow *dev_flow;
1872
1873         if (!flow)
1874                 return;
1875         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1876                 dv = &dev_flow->dv;
1877                 if (dv->flow) {
1878                         claim_zero(mlx5_glue->destroy_flow(dv->flow));
1879                         dv->flow = NULL;
1880                 }
1881                 if (dv->hrxq) {
1882                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1883                                 mlx5_hrxq_drop_release(dev);
1884                         else
1885                                 mlx5_hrxq_release(dev, dv->hrxq);
1886                         dv->hrxq = NULL;
1887                 }
1888         }
1889         if (flow->counter)
1890                 flow->counter = NULL;
1891 }
1892
1893 /**
1894  * Remove the flow from the NIC and the memory.
1895  *
1896  * @param[in] dev
1897  *   Pointer to the Ethernet device structure.
1898  * @param[in, out] flow
1899  *   Pointer to flow structure.
1900  */
1901 static void
1902 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1903 {
1904         struct mlx5_flow *dev_flow;
1905
1906         if (!flow)
1907                 return;
1908         flow_dv_remove(dev, flow);
1909         while (!LIST_EMPTY(&flow->dev_flows)) {
1910                 dev_flow = LIST_FIRST(&flow->dev_flows);
1911                 LIST_REMOVE(dev_flow, next);
1912                 if (dev_flow->dv.matcher)
1913                         flow_dv_matcher_release(dev, dev_flow);
1914                 if (dev_flow->dv.encap_decap_verbs_action) {
1915                         claim_zero(mlx5_glue->destroy_flow_action
1916                                 (dev_flow->dv.encap_decap_verbs_action));
1917                         dev_flow->dv.encap_decap_verbs_action = NULL;
1918                 }
1919                 rte_free(dev_flow);
1920         }
1921 }
1922
1923 /**
1924  * Query a flow.
1925  *
1926  * @see rte_flow_query()
1927  * @see rte_flow_ops
1928  */
1929 static int
1930 flow_dv_query(struct rte_eth_dev *dev __rte_unused,
1931               struct rte_flow *flow __rte_unused,
1932               const struct rte_flow_action *actions __rte_unused,
1933               void *data __rte_unused,
1934               struct rte_flow_error *error __rte_unused)
1935 {
1936         rte_errno = ENOTSUP;
1937         return -rte_errno;
1938 }
1939
1940
1941 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
1942         .validate = flow_dv_validate,
1943         .prepare = flow_dv_prepare,
1944         .translate = flow_dv_translate,
1945         .apply = flow_dv_apply,
1946         .remove = flow_dv_remove,
1947         .destroy = flow_dv_destroy,
1948         .query = flow_dv_query,
1949 };
1950
1951 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */