drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include "mlx5.h"
  30 #include "mlx5_defs.h"
  31 #include "mlx5_prm.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_flow.h"
  34
  35 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  36
  37 /**
  38  * Verify the @p attributes will be correctly understood by the NIC and store
  39  * them in the @p flow if everything is correct.
  40  *
  41  * @param[in] dev
  42  *   Pointer to dev struct.
  43  * @param[in] attributes
  44  *   Pointer to flow attributes
  45  * @param[out] error
  46  *   Pointer to error structure.
  47  *
  48  * @return
  49  *   0 on success, a negative errno value otherwise and rte_errno is set.
  50  */
  51 static int
  52 flow_dv_validate_attributes(struct rte_eth_dev *dev,
  53                             const struct rte_flow_attr *attributes,
  54                             struct rte_flow_error *error)
  55 {
  56         struct priv *priv = dev->data->dev_private;
  57         uint32_t priority_max = priv->config.flow_prio - 1;
  58
  59         if (attributes->group)
  60                 return rte_flow_error_set(error, ENOTSUP,
  61                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
  62                                           NULL,
  63                                           "groups is not supported");
  64         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
  65             attributes->priority >= priority_max)
  66                 return rte_flow_error_set(error, ENOTSUP,
  67                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
  68                                           NULL,
  69                                           "priority out of range");
  70         if (attributes->egress)
  71                 return rte_flow_error_set(error, ENOTSUP,
  72                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
  73                                           NULL,
  74                                           "egress is not supported");
  75         if (attributes->transfer)
  76                 return rte_flow_error_set(error, ENOTSUP,
  77                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
  78                                           NULL,
  79                                           "transfer is not supported");
  80         if (!attributes->ingress)
  81                 return rte_flow_error_set(error, EINVAL,
  82                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
  83                                           NULL,
  84                                           "ingress attribute is mandatory");
  85         return 0;
  86 }
  87
  88 /**
  89  * Internal validation function. For validating both actions and items.
  90  *
  91  * @param[in] dev
  92  *   Pointer to the rte_eth_dev structure.
  93  * @param[in] attr
  94  *   Pointer to the flow attributes.
  95  * @param[in] items
  96  *   Pointer to the list of items.
  97  * @param[in] actions
  98  *   Pointer to the list of actions.
  99  * @param[out] error
 100  *   Pointer to the error structure.
 101  *
 102  * @return
 103  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 104  */
 105 static int
 106 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 107                  const struct rte_flow_item items[],
 108                  const struct rte_flow_action actions[],
 109                  struct rte_flow_error *error)
 110 {
 111         int ret;
 112         uint32_t action_flags = 0;
 113         uint32_t item_flags = 0;
 114         int tunnel = 0;
 115         uint8_t next_protocol = 0xff;
 116         int actions_n = 0;
 117
 118         if (items == NULL)
 119                 return -1;
 120         ret = flow_dv_validate_attributes(dev, attr, error);
 121         if (ret < 0)
 122                 return ret;
 123         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 124                 switch (items->type) {
 125                 case RTE_FLOW_ITEM_TYPE_VOID:
 126                         break;
 127                 case RTE_FLOW_ITEM_TYPE_ETH:
 128                         ret = mlx5_flow_validate_item_eth(items, item_flags,
 129                                                           error);
 130                         if (ret < 0)
 131                                 return ret;
 132                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 133                                                MLX5_FLOW_LAYER_OUTER_L2;
 134                         break;
 135                 case RTE_FLOW_ITEM_TYPE_VLAN:
 136                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
 137                                                            error);
 138                         if (ret < 0)
 139                                 return ret;
 140                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
 141                                                MLX5_FLOW_LAYER_OUTER_VLAN;
 142                         break;
 143                 case RTE_FLOW_ITEM_TYPE_IPV4:
 144                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
 145                                                            error);
 146                         if (ret < 0)
 147                                 return ret;
 148                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 149                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 150                         if (items->mask != NULL &&
 151                             ((const struct rte_flow_item_ipv4 *)
 152                              items->mask)->hdr.next_proto_id)
 153                                 next_protocol =
 154                                         ((const struct rte_flow_item_ipv4 *)
 155                                          (items->spec))->hdr.next_proto_id;
 156                         break;
 157                 case RTE_FLOW_ITEM_TYPE_IPV6:
 158                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
 159                                                            error);
 160                         if (ret < 0)
 161                                 return ret;
 162                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 163                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 164                         if (items->mask != NULL &&
 165                             ((const struct rte_flow_item_ipv6 *)
 166                              items->mask)->hdr.proto)
 167                                 next_protocol =
 168                                         ((const struct rte_flow_item_ipv6 *)
 169                                          items->spec)->hdr.proto;
 170                         break;
 171                 case RTE_FLOW_ITEM_TYPE_UDP:
 172                         ret = mlx5_flow_validate_item_udp(items, item_flags,
 173                                                           next_protocol,
 174                                                           error);
 175                         if (ret < 0)
 176                                 return ret;
 177                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 178                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
 179                         break;
 180                 case RTE_FLOW_ITEM_TYPE_TCP:
 181                         ret = mlx5_flow_validate_item_tcp(items, item_flags,
 182                                                           next_protocol, error);
 183                         if (ret < 0)
 184                                 return ret;
 185                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 186                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
 187                         break;
 188                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 189                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
 190                                                             error);
 191                         if (ret < 0)
 192                                 return ret;
 193                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
 194                         break;
 195                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 196                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
 197                                                                 item_flags, dev,
 198                                                                 error);
 199                         if (ret < 0)
 200                                 return ret;
 201                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 202                         break;
 203                 case RTE_FLOW_ITEM_TYPE_GRE:
 204                         ret = mlx5_flow_validate_item_gre(items, item_flags,
 205                                                           next_protocol, error);
 206                         if (ret < 0)
 207                                 return ret;
 208                         item_flags |= MLX5_FLOW_LAYER_GRE;
 209                         break;
 210                 case RTE_FLOW_ITEM_TYPE_MPLS:
 211                         ret = mlx5_flow_validate_item_mpls(items, item_flags,
 212                                                            next_protocol,
 213                                                            error);
 214                         if (ret < 0)
 215                                 return ret;
 216                         item_flags |= MLX5_FLOW_LAYER_MPLS;
 217                         break;
 218                 default:
 219                         return rte_flow_error_set(error, ENOTSUP,
 220                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 221                                                   NULL, "item not supported");
 222                 }
 223         }
 224         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 225                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
 226                         return rte_flow_error_set(error, ENOTSUP,
 227                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 228                                                   actions, "too many actions");
 229                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 230                 switch (actions->type) {
 231                 case RTE_FLOW_ACTION_TYPE_VOID:
 232                         break;
 233                 case RTE_FLOW_ACTION_TYPE_FLAG:
 234                         ret = mlx5_flow_validate_action_flag(action_flags,
 235                                                              error);
 236                         if (ret < 0)
 237                                 return ret;
 238                         action_flags |= MLX5_FLOW_ACTION_FLAG;
 239                         ++actions_n;
 240                         break;
 241                 case RTE_FLOW_ACTION_TYPE_MARK:
 242                         ret = mlx5_flow_validate_action_mark(actions,
 243                                                              action_flags,
 244                                                              error);
 245                         if (ret < 0)
 246                                 return ret;
 247                         action_flags |= MLX5_FLOW_ACTION_MARK;
 248                         ++actions_n;
 249                         break;
 250                 case RTE_FLOW_ACTION_TYPE_DROP:
 251                         ret = mlx5_flow_validate_action_drop(action_flags,
 252                                                              error);
 253                         if (ret < 0)
 254                                 return ret;
 255                         action_flags |= MLX5_FLOW_ACTION_DROP;
 256                         ++actions_n;
 257                         break;
 258                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 259                         ret = mlx5_flow_validate_action_queue(actions,
 260                                                               action_flags, dev,
 261                                                               error);
 262                         if (ret < 0)
 263                                 return ret;
 264                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
 265                         ++actions_n;
 266                         break;
 267                 case RTE_FLOW_ACTION_TYPE_RSS:
 268                         ret = mlx5_flow_validate_action_rss(actions,
 269                                                             action_flags, dev,
 270                                                             error);
 271                         if (ret < 0)
 272                                 return ret;
 273                         action_flags |= MLX5_FLOW_ACTION_RSS;
 274                         ++actions_n;
 275                         break;
 276                 case RTE_FLOW_ACTION_TYPE_COUNT:
 277                         ret = mlx5_flow_validate_action_count(dev, error);
 278                         if (ret < 0)
 279                                 return ret;
 280                         action_flags |= MLX5_FLOW_ACTION_COUNT;
 281                         ++actions_n;
 282                         break;
 283                 default:
 284                         return rte_flow_error_set(error, ENOTSUP,
 285                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 286                                                   actions,
 287                                                   "action not supported");
 288                 }
 289         }
 290         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
 291                 return rte_flow_error_set(error, EINVAL,
 292                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
 293                                           "no fate action is found");
 294         return 0;
 295 }
 296
 297 /**
 298  * Internal preparation function. Allocates the DV flow size,
 299  * this size is constant.
 300  *
 301  * @param[in] attr
 302  *   Pointer to the flow attributes.
 303  * @param[in] items
 304  *   Pointer to the list of items.
 305  * @param[in] actions
 306  *   Pointer to the list of actions.
 307  * @param[out] item_flags
 308  *   Pointer to bit mask of all items detected.
 309  * @param[out] action_flags
 310  *   Pointer to bit mask of all actions detected.
 311  * @param[out] error
 312  *   Pointer to the error structure.
 313  *
 314  * @return
 315  *   Pointer to mlx5_flow object on success,
 316  *   otherwise NULL and rte_ernno is set.
 317  */
 318 static struct mlx5_flow *
 319 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
 320                 const struct rte_flow_item items[] __rte_unused,
 321                 const struct rte_flow_action actions[] __rte_unused,
 322                 uint64_t *item_flags __rte_unused,
 323                 uint64_t *action_flags __rte_unused,
 324                 struct rte_flow_error *error)
 325 {
 326         uint32_t size = sizeof(struct mlx5_flow);
 327         struct mlx5_flow *flow;
 328
 329         flow = rte_calloc(__func__, 1, size, 0);
 330         if (!flow) {
 331                 rte_flow_error_set(error, ENOMEM,
 332                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 333                                    "not enough memory to create flow");
 334                 return NULL;
 335         }
 336         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
 337         return flow;
 338 }
 339
 340 /**
 341  * Add Ethernet item to matcher and to the value.
 342  *
 343  * @param[in, out] matcher
 344  *   Flow matcher.
 345  * @param[in, out] key
 346  *   Flow matcher value.
 347  * @param[in] item
 348  *   Flow pattern to translate.
 349  * @param[in] inner
 350  *   Item is inner pattern.
 351  */
 352 static void
 353 flow_dv_translate_item_eth(void *matcher, void *key,
 354                            const struct rte_flow_item *item, int inner)
 355 {
 356         const struct rte_flow_item_eth *eth_m = item->mask;
 357         const struct rte_flow_item_eth *eth_v = item->spec;
 358         const struct rte_flow_item_eth nic_mask = {
 359                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 360                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 361                 .type = RTE_BE16(0xffff),
 362         };
 363         void *headers_m;
 364         void *headers_v;
 365         char *l24_v;
 366         unsigned int i;
 367
 368         if (!eth_v)
 369                 return;
 370         if (!eth_m)
 371                 eth_m = &nic_mask;
 372         if (inner) {
 373                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 374                                          inner_headers);
 375                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 376         } else {
 377                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 378                                          outer_headers);
 379                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 380         }
 381         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
 382                &eth_m->dst, sizeof(eth_m->dst));
 383         /* The value must be in the range of the mask. */
 384         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
 385         for (i = 0; i < sizeof(eth_m->dst); ++i)
 386                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
 387         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
 388                &eth_m->src, sizeof(eth_m->src));
 389         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
 390         /* The value must be in the range of the mask. */
 391         for (i = 0; i < sizeof(eth_m->dst); ++i)
 392                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
 393         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
 394                  rte_be_to_cpu_16(eth_m->type));
 395         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
 396         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
 397 }
 398
 399 /**
 400  * Add VLAN item to matcher and to the value.
 401  *
 402  * @param[in, out] matcher
 403  *   Flow matcher.
 404  * @param[in, out] key
 405  *   Flow matcher value.
 406  * @param[in] item
 407  *   Flow pattern to translate.
 408  * @param[in] inner
 409  *   Item is inner pattern.
 410  */
 411 static void
 412 flow_dv_translate_item_vlan(void *matcher, void *key,
 413                             const struct rte_flow_item *item,
 414                             int inner)
 415 {
 416         const struct rte_flow_item_vlan *vlan_m = item->mask;
 417         const struct rte_flow_item_vlan *vlan_v = item->spec;
 418         const struct rte_flow_item_vlan nic_mask = {
 419                 .tci = RTE_BE16(0x0fff),
 420                 .inner_type = RTE_BE16(0xffff),
 421         };
 422         void *headers_m;
 423         void *headers_v;
 424         uint16_t tci_m;
 425         uint16_t tci_v;
 426
 427         if (!vlan_v)
 428                 return;
 429         if (!vlan_m)
 430                 vlan_m = &nic_mask;
 431         if (inner) {
 432                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 433                                          inner_headers);
 434                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 435         } else {
 436                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 437                                          outer_headers);
 438                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 439         }
 440         tci_m = rte_be_to_cpu_16(vlan_m->tci);
 441         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
 442         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
 443         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
 444         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
 445         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
 446         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
 447         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
 448         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
 449         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
 450 }
 451
 452 /**
 453  * Add IPV4 item to matcher and to the value.
 454  *
 455  * @param[in, out] matcher
 456  *   Flow matcher.
 457  * @param[in, out] key
 458  *   Flow matcher value.
 459  * @param[in] item
 460  *   Flow pattern to translate.
 461  * @param[in] inner
 462  *   Item is inner pattern.
 463  */
 464 static void
 465 flow_dv_translate_item_ipv4(void *matcher, void *key,
 466                             const struct rte_flow_item *item,
 467                             int inner)
 468 {
 469         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
 470         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
 471         const struct rte_flow_item_ipv4 nic_mask = {
 472                 .hdr = {
 473                         .src_addr = RTE_BE32(0xffffffff),
 474                         .dst_addr = RTE_BE32(0xffffffff),
 475                         .type_of_service = 0xff,
 476                         .next_proto_id = 0xff,
 477                 },
 478         };
 479         void *headers_m;
 480         void *headers_v;
 481         char *l24_m;
 482         char *l24_v;
 483         uint8_t tos;
 484
 485         if (!ipv4_v)
 486                 return;
 487         if (!ipv4_m)
 488                 ipv4_m = &nic_mask;
 489         if (inner) {
 490                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 491                                          inner_headers);
 492                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 493         } else {
 494                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 495                                          outer_headers);
 496                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 497         }
 498         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 499         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
 500         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 501                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 502         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 503                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 504         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
 505         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
 506         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 507                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 508         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 509                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 510         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
 511         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
 512         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
 513         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
 514                  ipv4_m->hdr.type_of_service);
 515         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
 516         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
 517                  ipv4_m->hdr.type_of_service >> 2);
 518         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
 519         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 520                  ipv4_m->hdr.next_proto_id);
 521         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 522                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
 523 }
 524
 525 /**
 526  * Add IPV6 item to matcher and to the value.
 527  *
 528  * @param[in, out] matcher
 529  *   Flow matcher.
 530  * @param[in, out] key
 531  *   Flow matcher value.
 532  * @param[in] item
 533  *   Flow pattern to translate.
 534  * @param[in] inner
 535  *   Item is inner pattern.
 536  */
 537 static void
 538 flow_dv_translate_item_ipv6(void *matcher, void *key,
 539                             const struct rte_flow_item *item,
 540                             int inner)
 541 {
 542         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
 543         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
 544         const struct rte_flow_item_ipv6 nic_mask = {
 545                 .hdr = {
 546                         .src_addr =
 547                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
 548                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
 549                         .dst_addr =
 550                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
 551                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
 552                         .vtc_flow = RTE_BE32(0xffffffff),
 553                         .proto = 0xff,
 554                         .hop_limits = 0xff,
 555                 },
 556         };
 557         void *headers_m;
 558         void *headers_v;
 559         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 560         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 561         char *l24_m;
 562         char *l24_v;
 563         uint32_t vtc_m;
 564         uint32_t vtc_v;
 565         int i;
 566         int size;
 567
 568         if (!ipv6_v)
 569                 return;
 570         if (!ipv6_m)
 571                 ipv6_m = &nic_mask;
 572         if (inner) {
 573                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 574                                          inner_headers);
 575                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 576         } else {
 577                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 578                                          outer_headers);
 579                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 580         }
 581         size = sizeof(ipv6_m->hdr.dst_addr);
 582         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 583                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
 584         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 585                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
 586         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
 587         for (i = 0; i < size; ++i)
 588                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
 589         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 590                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
 591         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 592                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
 593         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
 594         for (i = 0; i < size; ++i)
 595                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
 596         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 597         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
 598         /* TOS. */
 599         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
 600         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
 601         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
 602         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
 603         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
 604         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
 605         /* Label. */
 606         if (inner) {
 607                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
 608                          vtc_m);
 609                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
 610                          vtc_v);
 611         } else {
 612                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
 613                          vtc_m);
 614                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
 615                          vtc_v);
 616         }
 617         /* Protocol. */
 618         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 619                  ipv6_m->hdr.proto);
 620         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 621                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
 622 }
 623
 624 /**
 625  * Add TCP item to matcher and to the value.
 626  *
 627  * @param[in, out] matcher
 628  *   Flow matcher.
 629  * @param[in, out] key
 630  *   Flow matcher value.
 631  * @param[in] item
 632  *   Flow pattern to translate.
 633  * @param[in] inner
 634  *   Item is inner pattern.
 635  */
 636 static void
 637 flow_dv_translate_item_tcp(void *matcher, void *key,
 638                            const struct rte_flow_item *item,
 639                            int inner)
 640 {
 641         const struct rte_flow_item_tcp *tcp_m = item->mask;
 642         const struct rte_flow_item_tcp *tcp_v = item->spec;
 643         void *headers_m;
 644         void *headers_v;
 645
 646         if (!tcp_v)
 647                 return;
 648         if (!tcp_m)
 649                 tcp_m = &rte_flow_item_tcp_mask;
 650         if (inner) {
 651                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 652                                          inner_headers);
 653                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 654         } else {
 655                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 656                                          outer_headers);
 657                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 658         }
 659         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
 660         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
 661         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
 662                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
 663         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
 664                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
 665         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
 666                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
 667         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
 668                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
 669 }
 670
 671 /**
 672  * Add UDP item to matcher and to the value.
 673  *
 674  * @param[in, out] matcher
 675  *   Flow matcher.
 676  * @param[in, out] key
 677  *   Flow matcher value.
 678  * @param[in] item
 679  *   Flow pattern to translate.
 680  * @param[in] inner
 681  *   Item is inner pattern.
 682  */
 683 static void
 684 flow_dv_translate_item_udp(void *matcher, void *key,
 685                            const struct rte_flow_item *item,
 686                            int inner)
 687 {
 688         const struct rte_flow_item_udp *udp_m = item->mask;
 689         const struct rte_flow_item_udp *udp_v = item->spec;
 690         void *headers_m;
 691         void *headers_v;
 692
 693         if (!udp_v)
 694                 return;
 695         if (!udp_m)
 696                 udp_m = &rte_flow_item_udp_mask;
 697         if (inner) {
 698                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 699                                          inner_headers);
 700                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 701         } else {
 702                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 703                                          outer_headers);
 704                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 705         }
 706         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
 707         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
 708         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
 709                  rte_be_to_cpu_16(udp_m->hdr.src_port));
 710         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
 711                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
 712         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
 713                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
 714         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
 715                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
 716 }
 717
 718 /**
 719  * Add GRE item to matcher and to the value.
 720  *
 721  * @param[in, out] matcher
 722  *   Flow matcher.
 723  * @param[in, out] key
 724  *   Flow matcher value.
 725  * @param[in] item
 726  *   Flow pattern to translate.
 727  * @param[in] inner
 728  *   Item is inner pattern.
 729  */
 730 static void
 731 flow_dv_translate_item_gre(void *matcher, void *key,
 732                            const struct rte_flow_item *item,
 733                            int inner)
 734 {
 735         const struct rte_flow_item_gre *gre_m = item->mask;
 736         const struct rte_flow_item_gre *gre_v = item->spec;
 737         void *headers_m;
 738         void *headers_v;
 739         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 740         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 741
 742         if (!gre_v)
 743                 return;
 744         if (!gre_m)
 745                 gre_m = &rte_flow_item_gre_mask;
 746         if (inner) {
 747                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 748                                          inner_headers);
 749                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 750         } else {
 751                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 752                                          outer_headers);
 753                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 754         }
 755         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
 756         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
 757         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
 758                  rte_be_to_cpu_16(gre_m->protocol));
 759         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
 760                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
 761 }
 762
 763 /**
 764  * Add NVGRE item to matcher and to the value.
 765  *
 766  * @param[in, out] matcher
 767  *   Flow matcher.
 768  * @param[in, out] key
 769  *   Flow matcher value.
 770  * @param[in] item
 771  *   Flow pattern to translate.
 772  * @param[in] inner
 773  *   Item is inner pattern.
 774  */
 775 static void
 776 flow_dv_translate_item_nvgre(void *matcher, void *key,
 777                              const struct rte_flow_item *item,
 778                              int inner)
 779 {
 780         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
 781         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
 782         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 783         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 784         char *gre_key_m;
 785         char *gre_key_v;
 786         int size;
 787         int i;
 788
 789         if (!nvgre_v)
 790                 return;
 791         if (!nvgre_m)
 792                 nvgre_m = &rte_flow_item_nvgre_mask;
 793         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
 794         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
 795         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
 796         memcpy(gre_key_m, nvgre_m->tni, size);
 797         for (i = 0; i < size; ++i)
 798                 gre_key_v[i] = gre_key_m[i] & ((const char *)(nvgre_v->tni))[i];
 799         flow_dv_translate_item_gre(matcher, key, item, inner);
 800 }
 801
 802 /**
 803  * Add VXLAN item to matcher and to the value.
 804  *
 805  * @param[in, out] matcher
 806  *   Flow matcher.
 807  * @param[in, out] key
 808  *   Flow matcher value.
 809  * @param[in] item
 810  *   Flow pattern to translate.
 811  * @param[in] inner
 812  *   Item is inner pattern.
 813  */
 814 static void
 815 flow_dv_translate_item_vxlan(void *matcher, void *key,
 816                              const struct rte_flow_item *item,
 817                              int inner)
 818 {
 819         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
 820         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
 821         void *headers_m;
 822         void *headers_v;
 823         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 824         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 825         char *vni_m;
 826         char *vni_v;
 827         uint16_t dport;
 828         int size;
 829         int i;
 830
 831         if (!vxlan_v)
 832                 return;
 833         if (!vxlan_m)
 834                 vxlan_m = &rte_flow_item_vxlan_mask;
 835         if (inner) {
 836                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 837                                          inner_headers);
 838                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 839         } else {
 840                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 841                                          outer_headers);
 842                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 843         }
 844         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
 845                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
 846         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
 847                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
 848                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
 849         }
 850         size = sizeof(vxlan_m->vni);
 851         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
 852         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
 853         memcpy(vni_m, vxlan_m->vni, size);
 854         for (i = 0; i < size; ++i)
 855                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
 856 }
 857
 858 /**
 859  * Update the matcher and the value based the selected item.
 860  *
 861  * @param[in, out] matcher
 862  *   Flow matcher.
 863  * @param[in, out] key
 864  *   Flow matcher value.
 865  * @param[in] item
 866  *   Flow pattern to translate.
 867  * @param[in, out] dev_flow
 868  *   Pointer to the mlx5_flow.
 869  * @param[in] inner
 870  *   Item is inner pattern.
 871  */
 872 static void
 873 flow_dv_create_item(void *matcher, void *key,
 874                     const struct rte_flow_item *item,
 875                     struct mlx5_flow *dev_flow,
 876                     int inner)
 877 {
 878         struct mlx5_flow_dv_matcher *tmatcher = matcher;
 879
 880         switch (item->type) {
 881         case RTE_FLOW_ITEM_TYPE_VOID:
 882         case RTE_FLOW_ITEM_TYPE_END:
 883                 break;
 884         case RTE_FLOW_ITEM_TYPE_ETH:
 885                 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
 886                                            inner);
 887                 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
 888                 break;
 889         case RTE_FLOW_ITEM_TYPE_VLAN:
 890                 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
 891                                             inner);
 892                 break;
 893         case RTE_FLOW_ITEM_TYPE_IPV4:
 894                 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
 895                                             inner);
 896                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
 897                 dev_flow->dv.hash_fields |=
 898                         mlx5_flow_hashfields_adjust(dev_flow, inner,
 899                                                     MLX5_IPV4_LAYER_TYPES,
 900                                                     MLX5_IPV4_IBV_RX_HASH);
 901                 break;
 902         case RTE_FLOW_ITEM_TYPE_IPV6:
 903                 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
 904                                             inner);
 905                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
 906                 dev_flow->dv.hash_fields |=
 907                         mlx5_flow_hashfields_adjust(dev_flow, inner,
 908                                                     MLX5_IPV6_LAYER_TYPES,
 909                                                     MLX5_IPV6_IBV_RX_HASH);
 910                 break;
 911         case RTE_FLOW_ITEM_TYPE_TCP:
 912                 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
 913                                            inner);
 914                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
 915                 dev_flow->dv.hash_fields |=
 916                         mlx5_flow_hashfields_adjust(dev_flow, inner,
 917                                                     ETH_RSS_TCP,
 918                                                     (IBV_RX_HASH_SRC_PORT_TCP |
 919                                                      IBV_RX_HASH_DST_PORT_TCP));
 920                 break;
 921         case RTE_FLOW_ITEM_TYPE_UDP:
 922                 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
 923                                            inner);
 924                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
 925                 dev_flow->verbs.hash_fields |=
 926                         mlx5_flow_hashfields_adjust(dev_flow, inner,
 927                                                     ETH_RSS_TCP,
 928                                                     (IBV_RX_HASH_SRC_PORT_TCP |
 929                                                      IBV_RX_HASH_DST_PORT_TCP));
 930                 break;
 931         case RTE_FLOW_ITEM_TYPE_NVGRE:
 932                 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
 933                                              inner);
 934                 break;
 935         case RTE_FLOW_ITEM_TYPE_GRE:
 936                 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
 937                                            inner);
 938                 break;
 939         case RTE_FLOW_ITEM_TYPE_VXLAN:
 940         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 941                 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
 942                                              inner);
 943                 break;
 944         default:
 945                 break;
 946         }
 947 }
 948
 949 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
 950
 951 #define HEADER_IS_ZERO(match_criteria, headers)                              \
 952         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
 953                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
 954
 955 /**
 956  * Calculate flow matcher enable bitmap.
 957  *
 958  * @param match_criteria
 959  *   Pointer to flow matcher criteria.
 960  *
 961  * @return
 962  *   Bitmap of enabled fields.
 963  */
 964 static uint8_t
 965 flow_dv_matcher_enable(uint32_t *match_criteria)
 966 {
 967         uint8_t match_criteria_enable;
 968
 969         match_criteria_enable =
 970                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
 971                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
 972         match_criteria_enable |=
 973                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
 974                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
 975         match_criteria_enable |=
 976                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
 977                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
 978         match_criteria_enable |=
 979                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
 980                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
 981
 982         return match_criteria_enable;
 983 }
 984
 985 /**
 986  * Register the flow matcher.
 987  *
 988  * @param dev[in, out]
 989  *   Pointer to rte_eth_dev structure.
 990  * @param[in, out] matcher
 991  *   Pointer to flow matcher.
 992  * @parm[in, out] dev_flow
 993  *   Pointer to the dev_flow.
 994  * @param[out] error
 995  *   pointer to error structure.
 996  *
 997  * @return
 998  *   0 on success otherwise -errno and errno is set.
 999  */
1000 static int
1001 flow_dv_matcher_register(struct rte_eth_dev *dev,
1002                          struct mlx5_flow_dv_matcher *matcher,
1003                          struct mlx5_flow *dev_flow,
1004                          struct rte_flow_error *error)
1005 {
1006         struct priv *priv = dev->data->dev_private;
1007         struct mlx5_flow_dv_matcher *cache_matcher;
1008         struct mlx5dv_flow_matcher_attr dv_attr = {
1009                 .type = IBV_FLOW_ATTR_NORMAL,
1010                 .match_mask = (void *)&matcher->mask,
1011         };
1012
1013         /* Lookup from cache. */
1014         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1015                 if (matcher->crc == cache_matcher->crc &&
1016                     matcher->priority == cache_matcher->priority &&
1017                     matcher->egress == cache_matcher->egress &&
1018                     !memcmp((const void *)matcher->mask.buf,
1019                             (const void *)cache_matcher->mask.buf,
1020                             cache_matcher->mask.size)) {
1021                         DRV_LOG(DEBUG,
1022                                 "priority %hd use %s matcher %p: refcnt %d++",
1023                                 cache_matcher->priority,
1024                                 cache_matcher->egress ? "tx" : "rx",
1025                                 (void *)cache_matcher,
1026                                 rte_atomic32_read(&cache_matcher->refcnt));
1027                         rte_atomic32_inc(&cache_matcher->refcnt);
1028                         dev_flow->dv.matcher = cache_matcher;
1029                         return 0;
1030                 }
1031         }
1032         /* Register new matcher. */
1033         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1034         if (!cache_matcher)
1035                 return rte_flow_error_set(error, ENOMEM,
1036                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1037                                           "cannot allocate matcher memory");
1038         *cache_matcher = *matcher;
1039         dv_attr.match_criteria_enable =
1040                 flow_dv_matcher_enable(cache_matcher->mask.buf);
1041         dv_attr.priority = matcher->priority;
1042         if (matcher->egress)
1043                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1044         cache_matcher->matcher_object =
1045                 mlx5dv_create_flow_matcher(priv->ctx, &dv_attr);
1046         if (!cache_matcher->matcher_object)
1047                 return rte_flow_error_set(error, ENOMEM,
1048                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1049                                           NULL, "cannot create matcher");
1050         rte_atomic32_inc(&cache_matcher->refcnt);
1051         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1052         dev_flow->dv.matcher = cache_matcher;
1053         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1054                 cache_matcher->priority,
1055                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1056                 rte_atomic32_read(&cache_matcher->refcnt));
1057         return 0;
1058 }
1059
1060
1061 /**
1062  * Fill the flow with DV spec.
1063  *
1064  * @param[in] dev
1065  *   Pointer to rte_eth_dev structure.
1066  * @param[in, out] dev_flow
1067  *   Pointer to the sub flow.
1068  * @param[in] attr
1069  *   Pointer to the flow attributes.
1070  * @param[in] items
1071  *   Pointer to the list of items.
1072  * @param[in] actions
1073  *   Pointer to the list of actions.
1074  * @param[out] error
1075  *   Pointer to the error structure.
1076  *
1077  * @return
1078  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1079  */
1080 static int
1081 flow_dv_translate(struct rte_eth_dev *dev,
1082                   struct mlx5_flow *dev_flow,
1083                   const struct rte_flow_attr *attr,
1084                   const struct rte_flow_item items[],
1085                   const struct rte_flow_action actions[] __rte_unused,
1086                   struct rte_flow_error *error)
1087 {
1088         struct priv *priv = dev->data->dev_private;
1089         uint64_t priority = attr->priority;
1090         struct mlx5_flow_dv_matcher matcher = {
1091                 .mask = {
1092                         .size = sizeof(matcher.mask.buf),
1093                 },
1094         };
1095         void *match_value = dev_flow->dv.value.buf;
1096         uint8_t inner = 0;
1097
1098         if (priority == MLX5_FLOW_PRIO_RSVD)
1099                 priority = priv->config.flow_prio - 1;
1100         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++)
1101                 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1102                                     inner);
1103         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1104                                      matcher.mask.size);
1105         if (priority == MLX5_FLOW_PRIO_RSVD)
1106                 priority = priv->config.flow_prio - 1;
1107         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1108                                                      matcher.priority);
1109         matcher.egress = attr->egress;
1110         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1111                 return -rte_errno;
1112         return 0;
1113 }
1114
1115 /**
1116  * Fills the flow_ops with the function pointers.
1117  *
1118  * @param[out] flow_ops
1119  *   Pointer to driver_ops structure.
1120  */
1121 void
1122 mlx5_flow_dv_get_driver_ops(struct mlx5_flow_driver_ops *flow_ops)
1123 {
1124         *flow_ops = (struct mlx5_flow_driver_ops) {
1125                 .validate = flow_dv_validate,
1126                 .prepare = flow_dv_prepare,
1127                 .translate = flow_dv_translate,
1128                 .apply = NULL,
1129                 .remove = NULL,
1130                 .destroy = NULL,
1131         };
1132 }
1133
1134 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */