drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_eth_ctrl.h>
  24 #include <rte_ethdev_driver.h>
  25 #include <rte_flow.h>
  26 #include <rte_flow_driver.h>
  27 #include <rte_malloc.h>
  28 #include <rte_ip.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  37
  38 /**
  39  * Validate META item.
  40  *
  41  * @param[in] dev
  42  *   Pointer to the rte_eth_dev structure.
  43  * @param[in] item
  44  *   Item specification.
  45  * @param[in] attr
  46  *   Attributes of flow that includes this item.
  47  * @param[out] error
  48  *   Pointer to error structure.
  49  *
  50  * @return
  51  *   0 on success, a negative errno value otherwise and rte_errno is set.
  52  */
  53 static int
  54 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
  55                            const struct rte_flow_item *item,
  56                            const struct rte_flow_attr *attr,
  57                            struct rte_flow_error *error)
  58 {
  59         const struct rte_flow_item_meta *spec = item->spec;
  60         const struct rte_flow_item_meta *mask = item->mask;
  61         const struct rte_flow_item_meta nic_mask = {
  62                 .data = RTE_BE32(UINT32_MAX)
  63         };
  64         int ret;
  65         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
  66
  67         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
  68                 return rte_flow_error_set(error, EPERM,
  69                                           RTE_FLOW_ERROR_TYPE_ITEM,
  70                                           NULL,
  71                                           "match on metadata offload "
  72                                           "configuration is off for this port");
  73         if (!spec)
  74                 return rte_flow_error_set(error, EINVAL,
  75                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  76                                           item->spec,
  77                                           "data cannot be empty");
  78         if (!spec->data)
  79                 return rte_flow_error_set(error, EINVAL,
  80                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
  81                                           NULL,
  82                                           "data cannot be zero");
  83         if (!mask)
  84                 mask = &rte_flow_item_meta_mask;
  85         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
  86                                         (const uint8_t *)&nic_mask,
  87                                         sizeof(struct rte_flow_item_meta),
  88                                         error);
  89         if (ret < 0)
  90                 return ret;
  91         if (attr->ingress)
  92                 return rte_flow_error_set(error, ENOTSUP,
  93                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
  94                                           NULL,
  95                                           "pattern not supported for ingress");
  96         return 0;
  97 }
  98
  99 /**
 100  * Verify the @p attributes will be correctly understood by the NIC and store
 101  * them in the @p flow if everything is correct.
 102  *
 103  * @param[in] dev
 104  *   Pointer to dev struct.
 105  * @param[in] attributes
 106  *   Pointer to flow attributes
 107  * @param[out] error
 108  *   Pointer to error structure.
 109  *
 110  * @return
 111  *   0 on success, a negative errno value otherwise and rte_errno is set.
 112  */
 113 static int
 114 flow_dv_validate_attributes(struct rte_eth_dev *dev,
 115                             const struct rte_flow_attr *attributes,
 116                             struct rte_flow_error *error)
 117 {
 118         struct priv *priv = dev->data->dev_private;
 119         uint32_t priority_max = priv->config.flow_prio - 1;
 120
 121         if (attributes->group)
 122                 return rte_flow_error_set(error, ENOTSUP,
 123                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 124                                           NULL,
 125                                           "groups is not supported");
 126         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
 127             attributes->priority >= priority_max)
 128                 return rte_flow_error_set(error, ENOTSUP,
 129                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 130                                           NULL,
 131                                           "priority out of range");
 132         if (attributes->transfer)
 133                 return rte_flow_error_set(error, ENOTSUP,
 134                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
 135                                           NULL,
 136                                           "transfer is not supported");
 137         if (!(attributes->egress ^ attributes->ingress))
 138                 return rte_flow_error_set(error, ENOTSUP,
 139                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 140                                           "must specify exactly one of "
 141                                           "ingress or egress");
 142         return 0;
 143 }
 144
 145 /**
 146  * Internal validation function. For validating both actions and items.
 147  *
 148  * @param[in] dev
 149  *   Pointer to the rte_eth_dev structure.
 150  * @param[in] attr
 151  *   Pointer to the flow attributes.
 152  * @param[in] items
 153  *   Pointer to the list of items.
 154  * @param[in] actions
 155  *   Pointer to the list of actions.
 156  * @param[out] error
 157  *   Pointer to the error structure.
 158  *
 159  * @return
 160  *   0 on success, a negative errno value otherwise and rte_ernno is set.
 161  */
 162 static int
 163 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 164                  const struct rte_flow_item items[],
 165                  const struct rte_flow_action actions[],
 166                  struct rte_flow_error *error)
 167 {
 168         int ret;
 169         uint32_t action_flags = 0;
 170         uint32_t item_flags = 0;
 171         int tunnel = 0;
 172         uint8_t next_protocol = 0xff;
 173         int actions_n = 0;
 174
 175         if (items == NULL)
 176                 return -1;
 177         ret = flow_dv_validate_attributes(dev, attr, error);
 178         if (ret < 0)
 179                 return ret;
 180         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 181                 switch (items->type) {
 182                 case RTE_FLOW_ITEM_TYPE_VOID:
 183                         break;
 184                 case RTE_FLOW_ITEM_TYPE_ETH:
 185                         ret = mlx5_flow_validate_item_eth(items, item_flags,
 186                                                           error);
 187                         if (ret < 0)
 188                                 return ret;
 189                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 190                                                MLX5_FLOW_LAYER_OUTER_L2;
 191                         break;
 192                 case RTE_FLOW_ITEM_TYPE_VLAN:
 193                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
 194                                                            error);
 195                         if (ret < 0)
 196                                 return ret;
 197                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
 198                                                MLX5_FLOW_LAYER_OUTER_VLAN;
 199                         break;
 200                 case RTE_FLOW_ITEM_TYPE_IPV4:
 201                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
 202                                                            error);
 203                         if (ret < 0)
 204                                 return ret;
 205                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 206                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 207                         if (items->mask != NULL &&
 208                             ((const struct rte_flow_item_ipv4 *)
 209                              items->mask)->hdr.next_proto_id)
 210                                 next_protocol =
 211                                         ((const struct rte_flow_item_ipv4 *)
 212                                          (items->spec))->hdr.next_proto_id;
 213                         break;
 214                 case RTE_FLOW_ITEM_TYPE_IPV6:
 215                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
 216                                                            error);
 217                         if (ret < 0)
 218                                 return ret;
 219                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 220                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 221                         if (items->mask != NULL &&
 222                             ((const struct rte_flow_item_ipv6 *)
 223                              items->mask)->hdr.proto)
 224                                 next_protocol =
 225                                         ((const struct rte_flow_item_ipv6 *)
 226                                          items->spec)->hdr.proto;
 227                         break;
 228                 case RTE_FLOW_ITEM_TYPE_UDP:
 229                         ret = mlx5_flow_validate_item_udp(items, item_flags,
 230                                                           next_protocol,
 231                                                           error);
 232                         if (ret < 0)
 233                                 return ret;
 234                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 235                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
 236                         break;
 237                 case RTE_FLOW_ITEM_TYPE_TCP:
 238                         ret = mlx5_flow_validate_item_tcp
 239                                                 (items, item_flags,
 240                                                  next_protocol,
 241                                                  &rte_flow_item_tcp_mask,
 242                                                  error);
 243                         if (ret < 0)
 244                                 return ret;
 245                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 246                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
 247                         break;
 248                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 249                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
 250                                                             error);
 251                         if (ret < 0)
 252                                 return ret;
 253                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
 254                         break;
 255                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 256                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
 257                                                                 item_flags, dev,
 258                                                                 error);
 259                         if (ret < 0)
 260                                 return ret;
 261                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 262                         break;
 263                 case RTE_FLOW_ITEM_TYPE_GRE:
 264                         ret = mlx5_flow_validate_item_gre(items, item_flags,
 265                                                           next_protocol, error);
 266                         if (ret < 0)
 267                                 return ret;
 268                         item_flags |= MLX5_FLOW_LAYER_GRE;
 269                         break;
 270                 case RTE_FLOW_ITEM_TYPE_MPLS:
 271                         ret = mlx5_flow_validate_item_mpls(items, item_flags,
 272                                                            next_protocol,
 273                                                            error);
 274                         if (ret < 0)
 275                                 return ret;
 276                         item_flags |= MLX5_FLOW_LAYER_MPLS;
 277                         break;
 278                 case RTE_FLOW_ITEM_TYPE_META:
 279                         ret = flow_dv_validate_item_meta(dev, items, attr,
 280                                                          error);
 281                         if (ret < 0)
 282                                 return ret;
 283                         item_flags |= MLX5_FLOW_ITEM_METADATA;
 284                         break;
 285                 default:
 286                         return rte_flow_error_set(error, ENOTSUP,
 287                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 288                                                   NULL, "item not supported");
 289                 }
 290         }
 291         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 292                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
 293                         return rte_flow_error_set(error, ENOTSUP,
 294                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 295                                                   actions, "too many actions");
 296                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 297                 switch (actions->type) {
 298                 case RTE_FLOW_ACTION_TYPE_VOID:
 299                         break;
 300                 case RTE_FLOW_ACTION_TYPE_FLAG:
 301                         ret = mlx5_flow_validate_action_flag(action_flags,
 302                                                              attr, error);
 303                         if (ret < 0)
 304                                 return ret;
 305                         action_flags |= MLX5_FLOW_ACTION_FLAG;
 306                         ++actions_n;
 307                         break;
 308                 case RTE_FLOW_ACTION_TYPE_MARK:
 309                         ret = mlx5_flow_validate_action_mark(actions,
 310                                                              action_flags,
 311                                                              attr, error);
 312                         if (ret < 0)
 313                                 return ret;
 314                         action_flags |= MLX5_FLOW_ACTION_MARK;
 315                         ++actions_n;
 316                         break;
 317                 case RTE_FLOW_ACTION_TYPE_DROP:
 318                         ret = mlx5_flow_validate_action_drop(action_flags,
 319                                                              attr, error);
 320                         if (ret < 0)
 321                                 return ret;
 322                         action_flags |= MLX5_FLOW_ACTION_DROP;
 323                         ++actions_n;
 324                         break;
 325                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 326                         ret = mlx5_flow_validate_action_queue(actions,
 327                                                               action_flags, dev,
 328                                                               attr, error);
 329                         if (ret < 0)
 330                                 return ret;
 331                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
 332                         ++actions_n;
 333                         break;
 334                 case RTE_FLOW_ACTION_TYPE_RSS:
 335                         ret = mlx5_flow_validate_action_rss(actions,
 336                                                             action_flags, dev,
 337                                                             attr, error);
 338                         if (ret < 0)
 339                                 return ret;
 340                         action_flags |= MLX5_FLOW_ACTION_RSS;
 341                         ++actions_n;
 342                         break;
 343                 case RTE_FLOW_ACTION_TYPE_COUNT:
 344                         ret = mlx5_flow_validate_action_count(dev, attr, error);
 345                         if (ret < 0)
 346                                 return ret;
 347                         action_flags |= MLX5_FLOW_ACTION_COUNT;
 348                         ++actions_n;
 349                         break;
 350                 default:
 351                         return rte_flow_error_set(error, ENOTSUP,
 352                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 353                                                   actions,
 354                                                   "action not supported");
 355                 }
 356         }
 357         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
 358                 return rte_flow_error_set(error, EINVAL,
 359                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
 360                                           "no fate action is found");
 361         return 0;
 362 }
 363
 364 /**
 365  * Internal preparation function. Allocates the DV flow size,
 366  * this size is constant.
 367  *
 368  * @param[in] attr
 369  *   Pointer to the flow attributes.
 370  * @param[in] items
 371  *   Pointer to the list of items.
 372  * @param[in] actions
 373  *   Pointer to the list of actions.
 374  * @param[out] item_flags
 375  *   Pointer to bit mask of all items detected.
 376  * @param[out] action_flags
 377  *   Pointer to bit mask of all actions detected.
 378  * @param[out] error
 379  *   Pointer to the error structure.
 380  *
 381  * @return
 382  *   Pointer to mlx5_flow object on success,
 383  *   otherwise NULL and rte_ernno is set.
 384  */
 385 static struct mlx5_flow *
 386 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
 387                 const struct rte_flow_item items[] __rte_unused,
 388                 const struct rte_flow_action actions[] __rte_unused,
 389                 uint64_t *item_flags __rte_unused,
 390                 uint64_t *action_flags __rte_unused,
 391                 struct rte_flow_error *error)
 392 {
 393         uint32_t size = sizeof(struct mlx5_flow);
 394         struct mlx5_flow *flow;
 395
 396         flow = rte_calloc(__func__, 1, size, 0);
 397         if (!flow) {
 398                 rte_flow_error_set(error, ENOMEM,
 399                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 400                                    "not enough memory to create flow");
 401                 return NULL;
 402         }
 403         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
 404         return flow;
 405 }
 406
 407 /**
 408  * Add Ethernet item to matcher and to the value.
 409  *
 410  * @param[in, out] matcher
 411  *   Flow matcher.
 412  * @param[in, out] key
 413  *   Flow matcher value.
 414  * @param[in] item
 415  *   Flow pattern to translate.
 416  * @param[in] inner
 417  *   Item is inner pattern.
 418  */
 419 static void
 420 flow_dv_translate_item_eth(void *matcher, void *key,
 421                            const struct rte_flow_item *item, int inner)
 422 {
 423         const struct rte_flow_item_eth *eth_m = item->mask;
 424         const struct rte_flow_item_eth *eth_v = item->spec;
 425         const struct rte_flow_item_eth nic_mask = {
 426                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 427                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 428                 .type = RTE_BE16(0xffff),
 429         };
 430         void *headers_m;
 431         void *headers_v;
 432         char *l24_v;
 433         unsigned int i;
 434
 435         if (!eth_v)
 436                 return;
 437         if (!eth_m)
 438                 eth_m = &nic_mask;
 439         if (inner) {
 440                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 441                                          inner_headers);
 442                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 443         } else {
 444                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 445                                          outer_headers);
 446                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 447         }
 448         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
 449                &eth_m->dst, sizeof(eth_m->dst));
 450         /* The value must be in the range of the mask. */
 451         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
 452         for (i = 0; i < sizeof(eth_m->dst); ++i)
 453                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
 454         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
 455                &eth_m->src, sizeof(eth_m->src));
 456         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
 457         /* The value must be in the range of the mask. */
 458         for (i = 0; i < sizeof(eth_m->dst); ++i)
 459                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
 460         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
 461                  rte_be_to_cpu_16(eth_m->type));
 462         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
 463         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
 464 }
 465
 466 /**
 467  * Add VLAN item to matcher and to the value.
 468  *
 469  * @param[in, out] matcher
 470  *   Flow matcher.
 471  * @param[in, out] key
 472  *   Flow matcher value.
 473  * @param[in] item
 474  *   Flow pattern to translate.
 475  * @param[in] inner
 476  *   Item is inner pattern.
 477  */
 478 static void
 479 flow_dv_translate_item_vlan(void *matcher, void *key,
 480                             const struct rte_flow_item *item,
 481                             int inner)
 482 {
 483         const struct rte_flow_item_vlan *vlan_m = item->mask;
 484         const struct rte_flow_item_vlan *vlan_v = item->spec;
 485         const struct rte_flow_item_vlan nic_mask = {
 486                 .tci = RTE_BE16(0x0fff),
 487                 .inner_type = RTE_BE16(0xffff),
 488         };
 489         void *headers_m;
 490         void *headers_v;
 491         uint16_t tci_m;
 492         uint16_t tci_v;
 493
 494         if (!vlan_v)
 495                 return;
 496         if (!vlan_m)
 497                 vlan_m = &nic_mask;
 498         if (inner) {
 499                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 500                                          inner_headers);
 501                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 502         } else {
 503                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 504                                          outer_headers);
 505                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 506         }
 507         tci_m = rte_be_to_cpu_16(vlan_m->tci);
 508         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
 509         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
 510         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
 511         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
 512         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
 513         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
 514         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
 515         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
 516         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
 517 }
 518
 519 /**
 520  * Add IPV4 item to matcher and to the value.
 521  *
 522  * @param[in, out] matcher
 523  *   Flow matcher.
 524  * @param[in, out] key
 525  *   Flow matcher value.
 526  * @param[in] item
 527  *   Flow pattern to translate.
 528  * @param[in] inner
 529  *   Item is inner pattern.
 530  */
 531 static void
 532 flow_dv_translate_item_ipv4(void *matcher, void *key,
 533                             const struct rte_flow_item *item,
 534                             int inner)
 535 {
 536         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
 537         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
 538         const struct rte_flow_item_ipv4 nic_mask = {
 539                 .hdr = {
 540                         .src_addr = RTE_BE32(0xffffffff),
 541                         .dst_addr = RTE_BE32(0xffffffff),
 542                         .type_of_service = 0xff,
 543                         .next_proto_id = 0xff,
 544                 },
 545         };
 546         void *headers_m;
 547         void *headers_v;
 548         char *l24_m;
 549         char *l24_v;
 550         uint8_t tos;
 551
 552         if (!ipv4_v)
 553                 return;
 554         if (!ipv4_m)
 555                 ipv4_m = &nic_mask;
 556         if (inner) {
 557                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 558                                          inner_headers);
 559                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 560         } else {
 561                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 562                                          outer_headers);
 563                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 564         }
 565         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 566         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
 567         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 568                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 569         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 570                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 571         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
 572         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
 573         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 574                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 575         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 576                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
 577         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
 578         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
 579         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
 580         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
 581                  ipv4_m->hdr.type_of_service);
 582         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
 583         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
 584                  ipv4_m->hdr.type_of_service >> 2);
 585         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
 586         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 587                  ipv4_m->hdr.next_proto_id);
 588         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 589                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
 590 }
 591
 592 /**
 593  * Add IPV6 item to matcher and to the value.
 594  *
 595  * @param[in, out] matcher
 596  *   Flow matcher.
 597  * @param[in, out] key
 598  *   Flow matcher value.
 599  * @param[in] item
 600  *   Flow pattern to translate.
 601  * @param[in] inner
 602  *   Item is inner pattern.
 603  */
 604 static void
 605 flow_dv_translate_item_ipv6(void *matcher, void *key,
 606                             const struct rte_flow_item *item,
 607                             int inner)
 608 {
 609         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
 610         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
 611         const struct rte_flow_item_ipv6 nic_mask = {
 612                 .hdr = {
 613                         .src_addr =
 614                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
 615                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
 616                         .dst_addr =
 617                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
 618                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
 619                         .vtc_flow = RTE_BE32(0xffffffff),
 620                         .proto = 0xff,
 621                         .hop_limits = 0xff,
 622                 },
 623         };
 624         void *headers_m;
 625         void *headers_v;
 626         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 627         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 628         char *l24_m;
 629         char *l24_v;
 630         uint32_t vtc_m;
 631         uint32_t vtc_v;
 632         int i;
 633         int size;
 634
 635         if (!ipv6_v)
 636                 return;
 637         if (!ipv6_m)
 638                 ipv6_m = &nic_mask;
 639         if (inner) {
 640                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 641                                          inner_headers);
 642                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 643         } else {
 644                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 645                                          outer_headers);
 646                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 647         }
 648         size = sizeof(ipv6_m->hdr.dst_addr);
 649         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 650                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
 651         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 652                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
 653         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
 654         for (i = 0; i < size; ++i)
 655                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
 656         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
 657                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
 658         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 659                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
 660         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
 661         for (i = 0; i < size; ++i)
 662                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
 663         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
 664         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
 665         /* TOS. */
 666         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
 667         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
 668         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
 669         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
 670         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
 671         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
 672         /* Label. */
 673         if (inner) {
 674                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
 675                          vtc_m);
 676                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
 677                          vtc_v);
 678         } else {
 679                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
 680                          vtc_m);
 681                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
 682                          vtc_v);
 683         }
 684         /* Protocol. */
 685         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 686                  ipv6_m->hdr.proto);
 687         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 688                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
 689 }
 690
 691 /**
 692  * Add TCP item to matcher and to the value.
 693  *
 694  * @param[in, out] matcher
 695  *   Flow matcher.
 696  * @param[in, out] key
 697  *   Flow matcher value.
 698  * @param[in] item
 699  *   Flow pattern to translate.
 700  * @param[in] inner
 701  *   Item is inner pattern.
 702  */
 703 static void
 704 flow_dv_translate_item_tcp(void *matcher, void *key,
 705                            const struct rte_flow_item *item,
 706                            int inner)
 707 {
 708         const struct rte_flow_item_tcp *tcp_m = item->mask;
 709         const struct rte_flow_item_tcp *tcp_v = item->spec;
 710         void *headers_m;
 711         void *headers_v;
 712
 713         if (!tcp_v)
 714                 return;
 715         if (!tcp_m)
 716                 tcp_m = &rte_flow_item_tcp_mask;
 717         if (inner) {
 718                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 719                                          inner_headers);
 720                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 721         } else {
 722                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 723                                          outer_headers);
 724                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 725         }
 726         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
 727         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
 728         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
 729                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
 730         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
 731                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
 732         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
 733                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
 734         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
 735                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
 736 }
 737
 738 /**
 739  * Add UDP item to matcher and to the value.
 740  *
 741  * @param[in, out] matcher
 742  *   Flow matcher.
 743  * @param[in, out] key
 744  *   Flow matcher value.
 745  * @param[in] item
 746  *   Flow pattern to translate.
 747  * @param[in] inner
 748  *   Item is inner pattern.
 749  */
 750 static void
 751 flow_dv_translate_item_udp(void *matcher, void *key,
 752                            const struct rte_flow_item *item,
 753                            int inner)
 754 {
 755         const struct rte_flow_item_udp *udp_m = item->mask;
 756         const struct rte_flow_item_udp *udp_v = item->spec;
 757         void *headers_m;
 758         void *headers_v;
 759
 760         if (!udp_v)
 761                 return;
 762         if (!udp_m)
 763                 udp_m = &rte_flow_item_udp_mask;
 764         if (inner) {
 765                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 766                                          inner_headers);
 767                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 768         } else {
 769                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 770                                          outer_headers);
 771                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 772         }
 773         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
 774         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
 775         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
 776                  rte_be_to_cpu_16(udp_m->hdr.src_port));
 777         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
 778                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
 779         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
 780                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
 781         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
 782                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
 783 }
 784
 785 /**
 786  * Add GRE item to matcher and to the value.
 787  *
 788  * @param[in, out] matcher
 789  *   Flow matcher.
 790  * @param[in, out] key
 791  *   Flow matcher value.
 792  * @param[in] item
 793  *   Flow pattern to translate.
 794  * @param[in] inner
 795  *   Item is inner pattern.
 796  */
 797 static void
 798 flow_dv_translate_item_gre(void *matcher, void *key,
 799                            const struct rte_flow_item *item,
 800                            int inner)
 801 {
 802         const struct rte_flow_item_gre *gre_m = item->mask;
 803         const struct rte_flow_item_gre *gre_v = item->spec;
 804         void *headers_m;
 805         void *headers_v;
 806         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 807         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 808
 809         if (!gre_v)
 810                 return;
 811         if (!gre_m)
 812                 gre_m = &rte_flow_item_gre_mask;
 813         if (inner) {
 814                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 815                                          inner_headers);
 816                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 817         } else {
 818                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 819                                          outer_headers);
 820                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 821         }
 822         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
 823         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
 824         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
 825                  rte_be_to_cpu_16(gre_m->protocol));
 826         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
 827                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
 828 }
 829
 830 /**
 831  * Add NVGRE item to matcher and to the value.
 832  *
 833  * @param[in, out] matcher
 834  *   Flow matcher.
 835  * @param[in, out] key
 836  *   Flow matcher value.
 837  * @param[in] item
 838  *   Flow pattern to translate.
 839  * @param[in] inner
 840  *   Item is inner pattern.
 841  */
 842 static void
 843 flow_dv_translate_item_nvgre(void *matcher, void *key,
 844                              const struct rte_flow_item *item,
 845                              int inner)
 846 {
 847         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
 848         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
 849         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 850         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 851         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
 852         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
 853         char *gre_key_m;
 854         char *gre_key_v;
 855         int size;
 856         int i;
 857
 858         if (!nvgre_v)
 859                 return;
 860         if (!nvgre_m)
 861                 nvgre_m = &rte_flow_item_nvgre_mask;
 862         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
 863         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
 864         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
 865         memcpy(gre_key_m, tni_flow_id_m, size);
 866         for (i = 0; i < size; ++i)
 867                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
 868         flow_dv_translate_item_gre(matcher, key, item, inner);
 869 }
 870
 871 /**
 872  * Add VXLAN item to matcher and to the value.
 873  *
 874  * @param[in, out] matcher
 875  *   Flow matcher.
 876  * @param[in, out] key
 877  *   Flow matcher value.
 878  * @param[in] item
 879  *   Flow pattern to translate.
 880  * @param[in] inner
 881  *   Item is inner pattern.
 882  */
 883 static void
 884 flow_dv_translate_item_vxlan(void *matcher, void *key,
 885                              const struct rte_flow_item *item,
 886                              int inner)
 887 {
 888         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
 889         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
 890         void *headers_m;
 891         void *headers_v;
 892         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
 893         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 894         char *vni_m;
 895         char *vni_v;
 896         uint16_t dport;
 897         int size;
 898         int i;
 899
 900         if (!vxlan_v)
 901                 return;
 902         if (!vxlan_m)
 903                 vxlan_m = &rte_flow_item_vxlan_mask;
 904         if (inner) {
 905                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 906                                          inner_headers);
 907                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
 908         } else {
 909                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
 910                                          outer_headers);
 911                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
 912         }
 913         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
 914                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
 915         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
 916                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
 917                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
 918         }
 919         size = sizeof(vxlan_m->vni);
 920         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
 921         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
 922         memcpy(vni_m, vxlan_m->vni, size);
 923         for (i = 0; i < size; ++i)
 924                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
 925 }
 926
 927 /**
 928  * Add META item to matcher
 929  *
 930  * @param[in, out] matcher
 931  *   Flow matcher.
 932  * @param[in, out] key
 933  *   Flow matcher value.
 934  * @param[in] item
 935  *   Flow pattern to translate.
 936  * @param[in] inner
 937  *   Item is inner pattern.
 938  */
 939 static void
 940 flow_dv_translate_item_meta(void *matcher, void *key,
 941                             const struct rte_flow_item *item)
 942 {
 943         const struct rte_flow_item_meta *meta_m;
 944         const struct rte_flow_item_meta *meta_v;
 945         void *misc2_m =
 946                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
 947         void *misc2_v =
 948                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
 949
 950         meta_m = (const void *)item->mask;
 951         if (!meta_m)
 952                 meta_m = &rte_flow_item_meta_mask;
 953         meta_v = (const void *)item->spec;
 954         if (meta_v) {
 955                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
 956                          rte_be_to_cpu_32(meta_m->data));
 957                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
 958                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
 959         }
 960 }
 961
 962 /**
 963  * Update the matcher and the value based the selected item.
 964  *
 965  * @param[in, out] matcher
 966  *   Flow matcher.
 967  * @param[in, out] key
 968  *   Flow matcher value.
 969  * @param[in] item
 970  *   Flow pattern to translate.
 971  * @param[in, out] dev_flow
 972  *   Pointer to the mlx5_flow.
 973  * @param[in] inner
 974  *   Item is inner pattern.
 975  */
 976 static void
 977 flow_dv_create_item(void *matcher, void *key,
 978                     const struct rte_flow_item *item,
 979                     struct mlx5_flow *dev_flow,
 980                     int inner)
 981 {
 982         struct mlx5_flow_dv_matcher *tmatcher = matcher;
 983
 984         switch (item->type) {
 985         case RTE_FLOW_ITEM_TYPE_VOID:
 986         case RTE_FLOW_ITEM_TYPE_END:
 987                 break;
 988         case RTE_FLOW_ITEM_TYPE_ETH:
 989                 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
 990                                            inner);
 991                 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
 992                 break;
 993         case RTE_FLOW_ITEM_TYPE_VLAN:
 994                 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
 995                                             inner);
 996                 break;
 997         case RTE_FLOW_ITEM_TYPE_IPV4:
 998                 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
 999                                             inner);
1000                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1001                 dev_flow->dv.hash_fields |=
1002                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1003                                                     MLX5_IPV4_LAYER_TYPES,
1004                                                     MLX5_IPV4_IBV_RX_HASH);
1005                 break;
1006         case RTE_FLOW_ITEM_TYPE_IPV6:
1007                 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
1008                                             inner);
1009                 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1010                 dev_flow->dv.hash_fields |=
1011                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1012                                                     MLX5_IPV6_LAYER_TYPES,
1013                                                     MLX5_IPV6_IBV_RX_HASH);
1014                 break;
1015         case RTE_FLOW_ITEM_TYPE_TCP:
1016                 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
1017                                            inner);
1018                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1019                 dev_flow->dv.hash_fields |=
1020                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1021                                                     ETH_RSS_TCP,
1022                                                     (IBV_RX_HASH_SRC_PORT_TCP |
1023                                                      IBV_RX_HASH_DST_PORT_TCP));
1024                 break;
1025         case RTE_FLOW_ITEM_TYPE_UDP:
1026                 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
1027                                            inner);
1028                 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1029                 dev_flow->verbs.hash_fields |=
1030                         mlx5_flow_hashfields_adjust(dev_flow, inner,
1031                                                     ETH_RSS_TCP,
1032                                                     (IBV_RX_HASH_SRC_PORT_TCP |
1033                                                      IBV_RX_HASH_DST_PORT_TCP));
1034                 break;
1035         case RTE_FLOW_ITEM_TYPE_NVGRE:
1036                 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
1037                                              inner);
1038                 break;
1039         case RTE_FLOW_ITEM_TYPE_GRE:
1040                 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
1041                                            inner);
1042                 break;
1043         case RTE_FLOW_ITEM_TYPE_VXLAN:
1044         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1045                 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
1046                                              inner);
1047                 break;
1048         case RTE_FLOW_ITEM_TYPE_META:
1049                 flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
1050                 break;
1051         default:
1052                 break;
1053         }
1054 }
1055
1056 /**
1057  * Store the requested actions in an array.
1058  *
1059  * @param[in] action
1060  *   Flow action to translate.
1061  * @param[in, out] dev_flow
1062  *   Pointer to the mlx5_flow.
1063  */
1064 static void
1065 flow_dv_create_action(const struct rte_flow_action *action,
1066                       struct mlx5_flow *dev_flow)
1067 {
1068         const struct rte_flow_action_queue *queue;
1069         const struct rte_flow_action_rss *rss;
1070         int actions_n = dev_flow->dv.actions_n;
1071         struct rte_flow *flow = dev_flow->flow;
1072
1073         switch (action->type) {
1074         case RTE_FLOW_ACTION_TYPE_VOID:
1075                 break;
1076         case RTE_FLOW_ACTION_TYPE_FLAG:
1077                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1078                 dev_flow->dv.actions[actions_n].tag_value =
1079                         MLX5_FLOW_MARK_DEFAULT;
1080                 actions_n++;
1081                 flow->actions |= MLX5_FLOW_ACTION_FLAG;
1082                 break;
1083         case RTE_FLOW_ACTION_TYPE_MARK:
1084                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1085                 dev_flow->dv.actions[actions_n].tag_value =
1086                         ((const struct rte_flow_action_mark *)
1087                          (action->conf))->id;
1088                 flow->actions |= MLX5_FLOW_ACTION_MARK;
1089                 actions_n++;
1090                 break;
1091         case RTE_FLOW_ACTION_TYPE_DROP:
1092                 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
1093                 flow->actions |= MLX5_FLOW_ACTION_DROP;
1094                 break;
1095         case RTE_FLOW_ACTION_TYPE_QUEUE:
1096                 queue = action->conf;
1097                 flow->rss.queue_num = 1;
1098                 (*flow->queue)[0] = queue->index;
1099                 flow->actions |= MLX5_FLOW_ACTION_QUEUE;
1100                 break;
1101         case RTE_FLOW_ACTION_TYPE_RSS:
1102                 rss = action->conf;
1103                 if (flow->queue)
1104                         memcpy((*flow->queue), rss->queue,
1105                                rss->queue_num * sizeof(uint16_t));
1106                 flow->rss.queue_num = rss->queue_num;
1107                 memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
1108                 flow->rss.types = rss->types;
1109                 flow->rss.level = rss->level;
1110                 /* Added to array only in apply since we need the QP */
1111                 flow->actions |= MLX5_FLOW_ACTION_RSS;
1112                 break;
1113         default:
1114                 break;
1115         }
1116         dev_flow->dv.actions_n = actions_n;
1117 }
1118
1119 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
1120
1121 #define HEADER_IS_ZERO(match_criteria, headers)                              \
1122         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
1123                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1124
1125 /**
1126  * Calculate flow matcher enable bitmap.
1127  *
1128  * @param match_criteria
1129  *   Pointer to flow matcher criteria.
1130  *
1131  * @return
1132  *   Bitmap of enabled fields.
1133  */
1134 static uint8_t
1135 flow_dv_matcher_enable(uint32_t *match_criteria)
1136 {
1137         uint8_t match_criteria_enable;
1138
1139         match_criteria_enable =
1140                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1141                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
1142         match_criteria_enable |=
1143                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1144                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
1145         match_criteria_enable |=
1146                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1147                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
1148         match_criteria_enable |=
1149                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
1150                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
1151
1152         return match_criteria_enable;
1153 }
1154
1155 /**
1156  * Register the flow matcher.
1157  *
1158  * @param dev[in, out]
1159  *   Pointer to rte_eth_dev structure.
1160  * @param[in, out] matcher
1161  *   Pointer to flow matcher.
1162  * @parm[in, out] dev_flow
1163  *   Pointer to the dev_flow.
1164  * @param[out] error
1165  *   pointer to error structure.
1166  *
1167  * @return
1168  *   0 on success otherwise -errno and errno is set.
1169  */
1170 static int
1171 flow_dv_matcher_register(struct rte_eth_dev *dev,
1172                          struct mlx5_flow_dv_matcher *matcher,
1173                          struct mlx5_flow *dev_flow,
1174                          struct rte_flow_error *error)
1175 {
1176         struct priv *priv = dev->data->dev_private;
1177         struct mlx5_flow_dv_matcher *cache_matcher;
1178         struct mlx5dv_flow_matcher_attr dv_attr = {
1179                 .type = IBV_FLOW_ATTR_NORMAL,
1180                 .match_mask = (void *)&matcher->mask,
1181         };
1182
1183         /* Lookup from cache. */
1184         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1185                 if (matcher->crc == cache_matcher->crc &&
1186                     matcher->priority == cache_matcher->priority &&
1187                     matcher->egress == cache_matcher->egress &&
1188                     !memcmp((const void *)matcher->mask.buf,
1189                             (const void *)cache_matcher->mask.buf,
1190                             cache_matcher->mask.size)) {
1191                         DRV_LOG(DEBUG,
1192                                 "priority %hd use %s matcher %p: refcnt %d++",
1193                                 cache_matcher->priority,
1194                                 cache_matcher->egress ? "tx" : "rx",
1195                                 (void *)cache_matcher,
1196                                 rte_atomic32_read(&cache_matcher->refcnt));
1197                         rte_atomic32_inc(&cache_matcher->refcnt);
1198                         dev_flow->dv.matcher = cache_matcher;
1199                         return 0;
1200                 }
1201         }
1202         /* Register new matcher. */
1203         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1204         if (!cache_matcher)
1205                 return rte_flow_error_set(error, ENOMEM,
1206                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1207                                           "cannot allocate matcher memory");
1208         *cache_matcher = *matcher;
1209         dv_attr.match_criteria_enable =
1210                 flow_dv_matcher_enable(cache_matcher->mask.buf);
1211         dv_attr.priority = matcher->priority;
1212         if (matcher->egress)
1213                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1214         cache_matcher->matcher_object =
1215                 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
1216         if (!cache_matcher->matcher_object)
1217                 return rte_flow_error_set(error, ENOMEM,
1218                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1219                                           NULL, "cannot create matcher");
1220         rte_atomic32_inc(&cache_matcher->refcnt);
1221         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1222         dev_flow->dv.matcher = cache_matcher;
1223         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1224                 cache_matcher->priority,
1225                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1226                 rte_atomic32_read(&cache_matcher->refcnt));
1227         return 0;
1228 }
1229
1230
1231 /**
1232  * Fill the flow with DV spec.
1233  *
1234  * @param[in] dev
1235  *   Pointer to rte_eth_dev structure.
1236  * @param[in, out] dev_flow
1237  *   Pointer to the sub flow.
1238  * @param[in] attr
1239  *   Pointer to the flow attributes.
1240  * @param[in] items
1241  *   Pointer to the list of items.
1242  * @param[in] actions
1243  *   Pointer to the list of actions.
1244  * @param[out] error
1245  *   Pointer to the error structure.
1246  *
1247  * @return
1248  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1249  */
1250 static int
1251 flow_dv_translate(struct rte_eth_dev *dev,
1252                   struct mlx5_flow *dev_flow,
1253                   const struct rte_flow_attr *attr,
1254                   const struct rte_flow_item items[],
1255                   const struct rte_flow_action actions[] __rte_unused,
1256                   struct rte_flow_error *error)
1257 {
1258         struct priv *priv = dev->data->dev_private;
1259         uint64_t priority = attr->priority;
1260         struct mlx5_flow_dv_matcher matcher = {
1261                 .mask = {
1262                         .size = sizeof(matcher.mask.buf),
1263                 },
1264         };
1265         void *match_value = dev_flow->dv.value.buf;
1266         uint8_t inner = 0;
1267
1268         if (priority == MLX5_FLOW_PRIO_RSVD)
1269                 priority = priv->config.flow_prio - 1;
1270         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++)
1271                 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1272                                     inner);
1273         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1274                                      matcher.mask.size);
1275         if (priority == MLX5_FLOW_PRIO_RSVD)
1276                 priority = priv->config.flow_prio - 1;
1277         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1278                                                      matcher.priority);
1279         matcher.egress = attr->egress;
1280         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1281                 return -rte_errno;
1282         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1283                 flow_dv_create_action(actions, dev_flow);
1284         return 0;
1285 }
1286
1287 /**
1288  * Apply the flow to the NIC.
1289  *
1290  * @param[in] dev
1291  *   Pointer to the Ethernet device structure.
1292  * @param[in, out] flow
1293  *   Pointer to flow structure.
1294  * @param[out] error
1295  *   Pointer to error structure.
1296  *
1297  * @return
1298  *   0 on success, a negative errno value otherwise and rte_errno is set.
1299  */
1300 static int
1301 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1302               struct rte_flow_error *error)
1303 {
1304         struct mlx5_flow_dv *dv;
1305         struct mlx5_flow *dev_flow;
1306         int n;
1307         int err;
1308
1309         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1310                 dv = &dev_flow->dv;
1311                 n = dv->actions_n;
1312                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1313                         dv->hrxq = mlx5_hrxq_drop_new(dev);
1314                         if (!dv->hrxq) {
1315                                 rte_flow_error_set
1316                                         (error, errno,
1317                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1318                                          "cannot get drop hash queue");
1319                                 goto error;
1320                         }
1321                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1322                         dv->actions[n].qp = dv->hrxq->qp;
1323                         n++;
1324                 } else if (flow->actions &
1325                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
1326                         struct mlx5_hrxq *hrxq;
1327                         hrxq = mlx5_hrxq_get(dev, flow->key,
1328                                              MLX5_RSS_HASH_KEY_LEN,
1329                                              dv->hash_fields,
1330                                              (*flow->queue),
1331                                              flow->rss.queue_num);
1332                         if (!hrxq)
1333                                 hrxq = mlx5_hrxq_new
1334                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
1335                                          dv->hash_fields, (*flow->queue),
1336                                          flow->rss.queue_num,
1337                                          !!(flow->layers &
1338                                             MLX5_FLOW_LAYER_TUNNEL));
1339                         if (!hrxq) {
1340                                 rte_flow_error_set
1341                                         (error, rte_errno,
1342                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1343                                          "cannot get hash queue");
1344                                 goto error;
1345                         }
1346                         dv->hrxq = hrxq;
1347                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1348                         dv->actions[n].qp = hrxq->qp;
1349                         n++;
1350                 }
1351                 dv->flow =
1352                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
1353                                                   (void *)&dv->value, n,
1354                                                   dv->actions);
1355                 if (!dv->flow) {
1356                         rte_flow_error_set(error, errno,
1357                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1358                                            NULL,
1359                                            "hardware refuses to create flow");
1360                         goto error;
1361                 }
1362         }
1363         return 0;
1364 error:
1365         err = rte_errno; /* Save rte_errno before cleanup. */
1366         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1367                 struct mlx5_flow_dv *dv = &dev_flow->dv;
1368                 if (dv->hrxq) {
1369                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1370                                 mlx5_hrxq_drop_release(dev);
1371                         else
1372                                 mlx5_hrxq_release(dev, dv->hrxq);
1373                         dv->hrxq = NULL;
1374                 }
1375         }
1376         rte_errno = err; /* Restore rte_errno. */
1377         return -rte_errno;
1378 }
1379
1380 /**
1381  * Release the flow matcher.
1382  *
1383  * @param dev
1384  *   Pointer to Ethernet device.
1385  * @param flow
1386  *   Pointer to mlx5_flow.
1387  *
1388  * @return
1389  *   1 while a reference on it exists, 0 when freed.
1390  */
1391 static int
1392 flow_dv_matcher_release(struct rte_eth_dev *dev,
1393                         struct mlx5_flow *flow)
1394 {
1395         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
1396
1397         assert(matcher->matcher_object);
1398         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
1399                 dev->data->port_id, (void *)matcher,
1400                 rte_atomic32_read(&matcher->refcnt));
1401         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
1402                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
1403                            (matcher->matcher_object));
1404                 LIST_REMOVE(matcher, next);
1405                 rte_free(matcher);
1406                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
1407                         dev->data->port_id, (void *)matcher);
1408                 return 0;
1409         }
1410         return 1;
1411 }
1412
1413 /**
1414  * Remove the flow from the NIC but keeps it in memory.
1415  *
1416  * @param[in] dev
1417  *   Pointer to Ethernet device.
1418  * @param[in, out] flow
1419  *   Pointer to flow structure.
1420  */
1421 static void
1422 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1423 {
1424         struct mlx5_flow_dv *dv;
1425         struct mlx5_flow *dev_flow;
1426
1427         if (!flow)
1428                 return;
1429         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1430                 dv = &dev_flow->dv;
1431                 if (dv->flow) {
1432                         claim_zero(mlx5_glue->destroy_flow(dv->flow));
1433                         dv->flow = NULL;
1434                 }
1435                 if (dv->hrxq) {
1436                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1437                                 mlx5_hrxq_drop_release(dev);
1438                         else
1439                                 mlx5_hrxq_release(dev, dv->hrxq);
1440                         dv->hrxq = NULL;
1441                 }
1442         }
1443         if (flow->counter)
1444                 flow->counter = NULL;
1445 }
1446
1447 /**
1448  * Remove the flow from the NIC and the memory.
1449  *
1450  * @param[in] dev
1451  *   Pointer to the Ethernet device structure.
1452  * @param[in, out] flow
1453  *   Pointer to flow structure.
1454  */
1455 static void
1456 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1457 {
1458         struct mlx5_flow *dev_flow;
1459
1460         if (!flow)
1461                 return;
1462         flow_dv_remove(dev, flow);
1463         while (!LIST_EMPTY(&flow->dev_flows)) {
1464                 dev_flow = LIST_FIRST(&flow->dev_flows);
1465                 LIST_REMOVE(dev_flow, next);
1466                 if (dev_flow->dv.matcher)
1467                         flow_dv_matcher_release(dev, dev_flow);
1468                 rte_free(dev_flow);
1469         }
1470 }
1471
1472 /**
1473  * Query a flow.
1474  *
1475  * @see rte_flow_query()
1476  * @see rte_flow_ops
1477  */
1478 static int
1479 flow_dv_query(struct rte_eth_dev *dev __rte_unused,
1480               struct rte_flow *flow __rte_unused,
1481               const struct rte_flow_action *actions __rte_unused,
1482               void *data __rte_unused,
1483               struct rte_flow_error *error __rte_unused)
1484 {
1485         rte_errno = ENOTSUP;
1486         return -rte_errno;
1487 }
1488
1489
1490 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
1491         .validate = flow_dv_validate,
1492         .prepare = flow_dv_prepare,
1493         .translate = flow_dv_translate,
1494         .apply = flow_dv_apply,
1495         .remove = flow_dv_remove,
1496         .destroy = flow_dv_destroy,
1497         .query = flow_dv_query,
1498 };
1499
1500 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */