drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  36
  37 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  38 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  39 #endif
  40
  41 union flow_dv_attr {
  42         struct {
  43                 uint32_t valid:1;
  44                 uint32_t ipv4:1;
  45                 uint32_t ipv6:1;
  46                 uint32_t tcp:1;
  47                 uint32_t udp:1;
  48                 uint32_t reserved:27;
  49         };
  50         uint32_t attr;
  51 };
  52
  53 /**
  54  * Initialize flow attributes structure according to flow items' types.
  55  *
  56  * @param[in] item
  57  *   Pointer to item specification.
  58  * @param[out] attr
  59  *   Pointer to flow attributes structure.
  60  */
  61 static void
  62 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  63 {
  64         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  65                 switch (item->type) {
  66                 case RTE_FLOW_ITEM_TYPE_IPV4:
  67                         attr->ipv4 = 1;
  68                         break;
  69                 case RTE_FLOW_ITEM_TYPE_IPV6:
  70                         attr->ipv6 = 1;
  71                         break;
  72                 case RTE_FLOW_ITEM_TYPE_UDP:
  73                         attr->udp = 1;
  74                         break;
  75                 case RTE_FLOW_ITEM_TYPE_TCP:
  76                         attr->tcp = 1;
  77                         break;
  78                 default:
  79                         break;
  80                 }
  81         }
  82         attr->valid = 1;
  83 }
  84
  85 struct field_modify_info {
  86         uint32_t size; /* Size of field in protocol header, in bytes. */
  87         uint32_t offset; /* Offset of field in protocol header, in bytes. */
  88         enum mlx5_modification_field id;
  89 };
  90
  91 struct field_modify_info modify_eth[] = {
  92         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
  93         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
  94         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
  95         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
  96         {0, 0, 0},
  97 };
  98
  99 struct field_modify_info modify_ipv4[] = {
 100         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 101         {4, 12, MLX5_MODI_OUT_SIPV4},
 102         {4, 16, MLX5_MODI_OUT_DIPV4},
 103         {0, 0, 0},
 104 };
 105
 106 struct field_modify_info modify_ipv6[] = {
 107         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 108         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 109         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 110         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 111         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 112         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 113         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 114         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 115         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 116         {0, 0, 0},
 117 };
 118
 119 struct field_modify_info modify_udp[] = {
 120         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 121         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 122         {0, 0, 0},
 123 };
 124
 125 struct field_modify_info modify_tcp[] = {
 126         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 127         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 128         {0, 0, 0},
 129 };
 130
 131 /**
 132  * Convert modify-header action to DV specification.
 133  *
 134  * @param[in] item
 135  *   Pointer to item specification.
 136  * @param[in] field
 137  *   Pointer to field modification information.
 138  * @param[in,out] resource
 139  *   Pointer to the modify-header resource.
 140  * @param[in] type
 141  *   Type of modification.
 142  * @param[out] error
 143  *   Pointer to the error structure.
 144  *
 145  * @return
 146  *   0 on success, a negative errno value otherwise and rte_errno is set.
 147  */
 148 static int
 149 flow_dv_convert_modify_action(struct rte_flow_item *item,
 150                               struct field_modify_info *field,
 151                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 152                               uint32_t type,
 153                               struct rte_flow_error *error)
 154 {
 155         uint32_t i = resource->actions_num;
 156         struct mlx5_modification_cmd *actions = resource->actions;
 157         const uint8_t *spec = item->spec;
 158         const uint8_t *mask = item->mask;
 159         uint32_t set;
 160
 161         while (field->size) {
 162                 set = 0;
 163                 /* Generate modify command for each mask segment. */
 164                 memcpy(&set, &mask[field->offset], field->size);
 165                 if (set) {
 166                         if (i >= MLX5_MODIFY_NUM)
 167                                 return rte_flow_error_set(error, EINVAL,
 168                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 169                                          "too many items to modify");
 170                         actions[i].action_type = type;
 171                         actions[i].field = field->id;
 172                         actions[i].length = field->size ==
 173                                         4 ? 0 : field->size * 8;
 174                         rte_memcpy(&actions[i].data[4 - field->size],
 175                                    &spec[field->offset], field->size);
 176                         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 177                         ++i;
 178                 }
 179                 if (resource->actions_num != i)
 180                         resource->actions_num = i;
 181                 field++;
 182         }
 183         if (!resource->actions_num)
 184                 return rte_flow_error_set(error, EINVAL,
 185                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 186                                           "invalid modification flow item");
 187         return 0;
 188 }
 189
 190 /**
 191  * Convert modify-header set IPv4 address action to DV specification.
 192  *
 193  * @param[in,out] resource
 194  *   Pointer to the modify-header resource.
 195  * @param[in] action
 196  *   Pointer to action specification.
 197  * @param[out] error
 198  *   Pointer to the error structure.
 199  *
 200  * @return
 201  *   0 on success, a negative errno value otherwise and rte_errno is set.
 202  */
 203 static int
 204 flow_dv_convert_action_modify_ipv4
 205                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 206                          const struct rte_flow_action *action,
 207                          struct rte_flow_error *error)
 208 {
 209         const struct rte_flow_action_set_ipv4 *conf =
 210                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 211         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 212         struct rte_flow_item_ipv4 ipv4;
 213         struct rte_flow_item_ipv4 ipv4_mask;
 214
 215         memset(&ipv4, 0, sizeof(ipv4));
 216         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 217         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 218                 ipv4.hdr.src_addr = conf->ipv4_addr;
 219                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 220         } else {
 221                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 222                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 223         }
 224         item.spec = &ipv4;
 225         item.mask = &ipv4_mask;
 226         return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
 227                                              MLX5_MODIFICATION_TYPE_SET, error);
 228 }
 229
 230 /**
 231  * Convert modify-header set IPv6 address action to DV specification.
 232  *
 233  * @param[in,out] resource
 234  *   Pointer to the modify-header resource.
 235  * @param[in] action
 236  *   Pointer to action specification.
 237  * @param[out] error
 238  *   Pointer to the error structure.
 239  *
 240  * @return
 241  *   0 on success, a negative errno value otherwise and rte_errno is set.
 242  */
 243 static int
 244 flow_dv_convert_action_modify_ipv6
 245                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 246                          const struct rte_flow_action *action,
 247                          struct rte_flow_error *error)
 248 {
 249         const struct rte_flow_action_set_ipv6 *conf =
 250                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 251         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 252         struct rte_flow_item_ipv6 ipv6;
 253         struct rte_flow_item_ipv6 ipv6_mask;
 254
 255         memset(&ipv6, 0, sizeof(ipv6));
 256         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 257         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 258                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 259                        sizeof(ipv6.hdr.src_addr));
 260                 memcpy(&ipv6_mask.hdr.src_addr,
 261                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 262                        sizeof(ipv6.hdr.src_addr));
 263         } else {
 264                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 265                        sizeof(ipv6.hdr.dst_addr));
 266                 memcpy(&ipv6_mask.hdr.dst_addr,
 267                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 268                        sizeof(ipv6.hdr.dst_addr));
 269         }
 270         item.spec = &ipv6;
 271         item.mask = &ipv6_mask;
 272         return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
 273                                              MLX5_MODIFICATION_TYPE_SET, error);
 274 }
 275
 276 /**
 277  * Convert modify-header set MAC address action to DV specification.
 278  *
 279  * @param[in,out] resource
 280  *   Pointer to the modify-header resource.
 281  * @param[in] action
 282  *   Pointer to action specification.
 283  * @param[out] error
 284  *   Pointer to the error structure.
 285  *
 286  * @return
 287  *   0 on success, a negative errno value otherwise and rte_errno is set.
 288  */
 289 static int
 290 flow_dv_convert_action_modify_mac
 291                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 292                          const struct rte_flow_action *action,
 293                          struct rte_flow_error *error)
 294 {
 295         const struct rte_flow_action_set_mac *conf =
 296                 (const struct rte_flow_action_set_mac *)(action->conf);
 297         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 298         struct rte_flow_item_eth eth;
 299         struct rte_flow_item_eth eth_mask;
 300
 301         memset(&eth, 0, sizeof(eth));
 302         memset(&eth_mask, 0, sizeof(eth_mask));
 303         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 304                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 305                        sizeof(eth.src.addr_bytes));
 306                 memcpy(&eth_mask.src.addr_bytes,
 307                        &rte_flow_item_eth_mask.src.addr_bytes,
 308                        sizeof(eth_mask.src.addr_bytes));
 309         } else {
 310                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 311                        sizeof(eth.dst.addr_bytes));
 312                 memcpy(&eth_mask.dst.addr_bytes,
 313                        &rte_flow_item_eth_mask.dst.addr_bytes,
 314                        sizeof(eth_mask.dst.addr_bytes));
 315         }
 316         item.spec = &eth;
 317         item.mask = &eth_mask;
 318         return flow_dv_convert_modify_action(&item, modify_eth, resource,
 319                                              MLX5_MODIFICATION_TYPE_SET, error);
 320 }
 321
 322 /**
 323  * Convert modify-header set TP action to DV specification.
 324  *
 325  * @param[in,out] resource
 326  *   Pointer to the modify-header resource.
 327  * @param[in] action
 328  *   Pointer to action specification.
 329  * @param[in] items
 330  *   Pointer to rte_flow_item objects list.
 331  * @param[in] attr
 332  *   Pointer to flow attributes structure.
 333  * @param[out] error
 334  *   Pointer to the error structure.
 335  *
 336  * @return
 337  *   0 on success, a negative errno value otherwise and rte_errno is set.
 338  */
 339 static int
 340 flow_dv_convert_action_modify_tp
 341                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 342                          const struct rte_flow_action *action,
 343                          const struct rte_flow_item *items,
 344                          union flow_dv_attr *attr,
 345                          struct rte_flow_error *error)
 346 {
 347         const struct rte_flow_action_set_tp *conf =
 348                 (const struct rte_flow_action_set_tp *)(action->conf);
 349         struct rte_flow_item item;
 350         struct rte_flow_item_udp udp;
 351         struct rte_flow_item_udp udp_mask;
 352         struct rte_flow_item_tcp tcp;
 353         struct rte_flow_item_tcp tcp_mask;
 354         struct field_modify_info *field;
 355
 356         if (!attr->valid)
 357                 flow_dv_attr_init(items, attr);
 358         if (attr->udp) {
 359                 memset(&udp, 0, sizeof(udp));
 360                 memset(&udp_mask, 0, sizeof(udp_mask));
 361                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 362                         udp.hdr.src_port = conf->port;
 363                         udp_mask.hdr.src_port =
 364                                         rte_flow_item_udp_mask.hdr.src_port;
 365                 } else {
 366                         udp.hdr.dst_port = conf->port;
 367                         udp_mask.hdr.dst_port =
 368                                         rte_flow_item_udp_mask.hdr.dst_port;
 369                 }
 370                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 371                 item.spec = &udp;
 372                 item.mask = &udp_mask;
 373                 field = modify_udp;
 374         }
 375         if (attr->tcp) {
 376                 memset(&tcp, 0, sizeof(tcp));
 377                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 378                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 379                         tcp.hdr.src_port = conf->port;
 380                         tcp_mask.hdr.src_port =
 381                                         rte_flow_item_tcp_mask.hdr.src_port;
 382                 } else {
 383                         tcp.hdr.dst_port = conf->port;
 384                         tcp_mask.hdr.dst_port =
 385                                         rte_flow_item_tcp_mask.hdr.dst_port;
 386                 }
 387                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 388                 item.spec = &tcp;
 389                 item.mask = &tcp_mask;
 390                 field = modify_tcp;
 391         }
 392         return flow_dv_convert_modify_action(&item, field, resource,
 393                                              MLX5_MODIFICATION_TYPE_SET, error);
 394 }
 395
 396 /**
 397  * Convert modify-header set TTL action to DV specification.
 398  *
 399  * @param[in,out] resource
 400  *   Pointer to the modify-header resource.
 401  * @param[in] action
 402  *   Pointer to action specification.
 403  * @param[in] items
 404  *   Pointer to rte_flow_item objects list.
 405  * @param[in] attr
 406  *   Pointer to flow attributes structure.
 407  * @param[out] error
 408  *   Pointer to the error structure.
 409  *
 410  * @return
 411  *   0 on success, a negative errno value otherwise and rte_errno is set.
 412  */
 413 static int
 414 flow_dv_convert_action_modify_ttl
 415                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 416                          const struct rte_flow_action *action,
 417                          const struct rte_flow_item *items,
 418                          union flow_dv_attr *attr,
 419                          struct rte_flow_error *error)
 420 {
 421         const struct rte_flow_action_set_ttl *conf =
 422                 (const struct rte_flow_action_set_ttl *)(action->conf);
 423         struct rte_flow_item item;
 424         struct rte_flow_item_ipv4 ipv4;
 425         struct rte_flow_item_ipv4 ipv4_mask;
 426         struct rte_flow_item_ipv6 ipv6;
 427         struct rte_flow_item_ipv6 ipv6_mask;
 428         struct field_modify_info *field;
 429
 430         if (!attr->valid)
 431                 flow_dv_attr_init(items, attr);
 432         if (attr->ipv4) {
 433                 memset(&ipv4, 0, sizeof(ipv4));
 434                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 435                 ipv4.hdr.time_to_live = conf->ttl_value;
 436                 ipv4_mask.hdr.time_to_live = 0xFF;
 437                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 438                 item.spec = &ipv4;
 439                 item.mask = &ipv4_mask;
 440                 field = modify_ipv4;
 441         }
 442         if (attr->ipv6) {
 443                 memset(&ipv6, 0, sizeof(ipv6));
 444                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 445                 ipv6.hdr.hop_limits = conf->ttl_value;
 446                 ipv6_mask.hdr.hop_limits = 0xFF;
 447                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 448                 item.spec = &ipv6;
 449                 item.mask = &ipv6_mask;
 450                 field = modify_ipv6;
 451         }
 452         return flow_dv_convert_modify_action(&item, field, resource,
 453                                              MLX5_MODIFICATION_TYPE_SET, error);
 454 }
 455
 456 /**
 457  * Convert modify-header decrement TTL action to DV specification.
 458  *
 459  * @param[in,out] resource
 460  *   Pointer to the modify-header resource.
 461  * @param[in] action
 462  *   Pointer to action specification.
 463  * @param[in] items
 464  *   Pointer to rte_flow_item objects list.
 465  * @param[in] attr
 466  *   Pointer to flow attributes structure.
 467  * @param[out] error
 468  *   Pointer to the error structure.
 469  *
 470  * @return
 471  *   0 on success, a negative errno value otherwise and rte_errno is set.
 472  */
 473 static int
 474 flow_dv_convert_action_modify_dec_ttl
 475                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 476                          const struct rte_flow_item *items,
 477                          union flow_dv_attr *attr,
 478                          struct rte_flow_error *error)
 479 {
 480         struct rte_flow_item item;
 481         struct rte_flow_item_ipv4 ipv4;
 482         struct rte_flow_item_ipv4 ipv4_mask;
 483         struct rte_flow_item_ipv6 ipv6;
 484         struct rte_flow_item_ipv6 ipv6_mask;
 485         struct field_modify_info *field;
 486
 487         if (!attr->valid)
 488                 flow_dv_attr_init(items, attr);
 489         if (attr->ipv4) {
 490                 memset(&ipv4, 0, sizeof(ipv4));
 491                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 492                 ipv4.hdr.time_to_live = 0xFF;
 493                 ipv4_mask.hdr.time_to_live = 0xFF;
 494                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 495                 item.spec = &ipv4;
 496                 item.mask = &ipv4_mask;
 497                 field = modify_ipv4;
 498         }
 499         if (attr->ipv6) {
 500                 memset(&ipv6, 0, sizeof(ipv6));
 501                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 502                 ipv6.hdr.hop_limits = 0xFF;
 503                 ipv6_mask.hdr.hop_limits = 0xFF;
 504                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 505                 item.spec = &ipv6;
 506                 item.mask = &ipv6_mask;
 507                 field = modify_ipv6;
 508         }
 509         return flow_dv_convert_modify_action(&item, field, resource,
 510                                              MLX5_MODIFICATION_TYPE_ADD, error);
 511 }
 512
 513 /**
 514  * Validate META item.
 515  *
 516  * @param[in] dev
 517  *   Pointer to the rte_eth_dev structure.
 518  * @param[in] item
 519  *   Item specification.
 520  * @param[in] attr
 521  *   Attributes of flow that includes this item.
 522  * @param[out] error
 523  *   Pointer to error structure.
 524  *
 525  * @return
 526  *   0 on success, a negative errno value otherwise and rte_errno is set.
 527  */
 528 static int
 529 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
 530                            const struct rte_flow_item *item,
 531                            const struct rte_flow_attr *attr,
 532                            struct rte_flow_error *error)
 533 {
 534         const struct rte_flow_item_meta *spec = item->spec;
 535         const struct rte_flow_item_meta *mask = item->mask;
 536         const struct rte_flow_item_meta nic_mask = {
 537                 .data = RTE_BE32(UINT32_MAX)
 538         };
 539         int ret;
 540         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
 541
 542         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
 543                 return rte_flow_error_set(error, EPERM,
 544                                           RTE_FLOW_ERROR_TYPE_ITEM,
 545                                           NULL,
 546                                           "match on metadata offload "
 547                                           "configuration is off for this port");
 548         if (!spec)
 549                 return rte_flow_error_set(error, EINVAL,
 550                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 551                                           item->spec,
 552                                           "data cannot be empty");
 553         if (!spec->data)
 554                 return rte_flow_error_set(error, EINVAL,
 555                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 556                                           NULL,
 557                                           "data cannot be zero");
 558         if (!mask)
 559                 mask = &rte_flow_item_meta_mask;
 560         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 561                                         (const uint8_t *)&nic_mask,
 562                                         sizeof(struct rte_flow_item_meta),
 563                                         error);
 564         if (ret < 0)
 565                 return ret;
 566         if (attr->ingress)
 567                 return rte_flow_error_set(error, ENOTSUP,
 568                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 569                                           NULL,
 570                                           "pattern not supported for ingress");
 571         return 0;
 572 }
 573
 574 /**
 575  * Validate count action.
 576  *
 577  * @param[in] dev
 578  *   device otr.
 579  * @param[out] error
 580  *   Pointer to error structure.
 581  *
 582  * @return
 583  *   0 on success, a negative errno value otherwise and rte_errno is set.
 584  */
 585 static int
 586 flow_dv_validate_action_count(struct rte_eth_dev *dev,
 587                               struct rte_flow_error *error)
 588 {
 589         struct mlx5_priv *priv = dev->data->dev_private;
 590
 591         if (!priv->config.devx)
 592                 goto notsup_err;
 593 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
 594         return 0;
 595 #endif
 596 notsup_err:
 597         return rte_flow_error_set
 598                       (error, ENOTSUP,
 599                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 600                        NULL,
 601                        "count action not supported");
 602 }
 603
 604 /**
 605  * Validate the L2 encap action.
 606  *
 607  * @param[in] action_flags
 608  *   Holds the actions detected until now.
 609  * @param[in] action
 610  *   Pointer to the encap action.
 611  * @param[in] attr
 612  *   Pointer to flow attributes
 613  * @param[out] error
 614  *   Pointer to error structure.
 615  *
 616  * @return
 617  *   0 on success, a negative errno value otherwise and rte_errno is set.
 618  */
 619 static int
 620 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 621                                  const struct rte_flow_action *action,
 622                                  const struct rte_flow_attr *attr,
 623                                  struct rte_flow_error *error)
 624 {
 625         if (!(action->conf))
 626                 return rte_flow_error_set(error, EINVAL,
 627                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 628                                           "configuration cannot be null");
 629         if (action_flags & MLX5_FLOW_ACTION_DROP)
 630                 return rte_flow_error_set(error, EINVAL,
 631                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 632                                           "can't drop and encap in same flow");
 633         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 634                 return rte_flow_error_set(error, EINVAL,
 635                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 636                                           "can only have a single encap or"
 637                                           " decap action in a flow");
 638         if (attr->ingress)
 639                 return rte_flow_error_set(error, ENOTSUP,
 640                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 641                                           NULL,
 642                                           "encap action not supported for "
 643                                           "ingress");
 644         return 0;
 645 }
 646
 647 /**
 648  * Validate the L2 decap action.
 649  *
 650  * @param[in] action_flags
 651  *   Holds the actions detected until now.
 652  * @param[in] attr
 653  *   Pointer to flow attributes
 654  * @param[out] error
 655  *   Pointer to error structure.
 656  *
 657  * @return
 658  *   0 on success, a negative errno value otherwise and rte_errno is set.
 659  */
 660 static int
 661 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 662                                  const struct rte_flow_attr *attr,
 663                                  struct rte_flow_error *error)
 664 {
 665         if (action_flags & MLX5_FLOW_ACTION_DROP)
 666                 return rte_flow_error_set(error, EINVAL,
 667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 668                                           "can't drop and decap in same flow");
 669         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 670                 return rte_flow_error_set(error, EINVAL,
 671                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 672                                           "can only have a single encap or"
 673                                           " decap action in a flow");
 674         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 675                 return rte_flow_error_set(error, EINVAL,
 676                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 677                                           "can't have decap action after"
 678                                           " modify action");
 679         if (attr->egress)
 680                 return rte_flow_error_set(error, ENOTSUP,
 681                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 682                                           NULL,
 683                                           "decap action not supported for "
 684                                           "egress");
 685         return 0;
 686 }
 687
 688 /**
 689  * Validate the raw encap action.
 690  *
 691  * @param[in] action_flags
 692  *   Holds the actions detected until now.
 693  * @param[in] action
 694  *   Pointer to the encap action.
 695  * @param[in] attr
 696  *   Pointer to flow attributes
 697  * @param[out] error
 698  *   Pointer to error structure.
 699  *
 700  * @return
 701  *   0 on success, a negative errno value otherwise and rte_errno is set.
 702  */
 703 static int
 704 flow_dv_validate_action_raw_encap(uint64_t action_flags,
 705                                   const struct rte_flow_action *action,
 706                                   const struct rte_flow_attr *attr,
 707                                   struct rte_flow_error *error)
 708 {
 709         if (!(action->conf))
 710                 return rte_flow_error_set(error, EINVAL,
 711                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 712                                           "configuration cannot be null");
 713         if (action_flags & MLX5_FLOW_ACTION_DROP)
 714                 return rte_flow_error_set(error, EINVAL,
 715                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 716                                           "can't drop and encap in same flow");
 717         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 718                 return rte_flow_error_set(error, EINVAL,
 719                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 720                                           "can only have a single encap"
 721                                           " action in a flow");
 722         /* encap without preceding decap is not supported for ingress */
 723         if (attr->ingress && !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
 724                 return rte_flow_error_set(error, ENOTSUP,
 725                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 726                                           NULL,
 727                                           "encap action not supported for "
 728                                           "ingress");
 729         return 0;
 730 }
 731
 732 /**
 733  * Validate the raw decap action.
 734  *
 735  * @param[in] action_flags
 736  *   Holds the actions detected until now.
 737  * @param[in] action
 738  *   Pointer to the encap action.
 739  * @param[in] attr
 740  *   Pointer to flow attributes
 741  * @param[out] error
 742  *   Pointer to error structure.
 743  *
 744  * @return
 745  *   0 on success, a negative errno value otherwise and rte_errno is set.
 746  */
 747 static int
 748 flow_dv_validate_action_raw_decap(uint64_t action_flags,
 749                                   const struct rte_flow_action *action,
 750                                   const struct rte_flow_attr *attr,
 751                                   struct rte_flow_error *error)
 752 {
 753         if (action_flags & MLX5_FLOW_ACTION_DROP)
 754                 return rte_flow_error_set(error, EINVAL,
 755                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 756                                           "can't drop and decap in same flow");
 757         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 758                 return rte_flow_error_set(error, EINVAL,
 759                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 760                                           "can't have encap action before"
 761                                           " decap action");
 762         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
 763                 return rte_flow_error_set(error, EINVAL,
 764                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 765                                           "can only have a single decap"
 766                                           " action in a flow");
 767         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 768                 return rte_flow_error_set(error, EINVAL,
 769                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 770                                           "can't have decap action after"
 771                                           " modify action");
 772         /* decap action is valid on egress only if it is followed by encap */
 773         if (attr->egress) {
 774                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
 775                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
 776                        action++) {
 777                 }
 778                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
 779                         return rte_flow_error_set
 780                                         (error, ENOTSUP,
 781                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 782                                          NULL, "decap action not supported"
 783                                          " for egress");
 784         }
 785         return 0;
 786 }
 787
 788 /**
 789  * Find existing encap/decap resource or create and register a new one.
 790  *
 791  * @param dev[in, out]
 792  *   Pointer to rte_eth_dev structure.
 793  * @param[in, out] resource
 794  *   Pointer to encap/decap resource.
 795  * @parm[in, out] dev_flow
 796  *   Pointer to the dev_flow.
 797  * @param[out] error
 798  *   pointer to error structure.
 799  *
 800  * @return
 801  *   0 on success otherwise -errno and errno is set.
 802  */
 803 static int
 804 flow_dv_encap_decap_resource_register
 805                         (struct rte_eth_dev *dev,
 806                          struct mlx5_flow_dv_encap_decap_resource *resource,
 807                          struct mlx5_flow *dev_flow,
 808                          struct rte_flow_error *error)
 809 {
 810         struct mlx5_priv *priv = dev->data->dev_private;
 811         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
 812         struct rte_flow *flow = dev_flow->flow;
 813         struct mlx5dv_dr_ns *ns;
 814
 815         resource->flags = flow->group ? 0 : 1;
 816         if (flow->ingress)
 817                 ns = priv->rx_ns;
 818         else
 819                 ns = priv->tx_ns;
 820
 821         /* Lookup a matching resource from cache. */
 822         LIST_FOREACH(cache_resource, &priv->encaps_decaps, next) {
 823                 if (resource->reformat_type == cache_resource->reformat_type &&
 824                     resource->ft_type == cache_resource->ft_type &&
 825                     resource->flags == cache_resource->flags &&
 826                     resource->size == cache_resource->size &&
 827                     !memcmp((const void *)resource->buf,
 828                             (const void *)cache_resource->buf,
 829                             resource->size)) {
 830                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
 831                                 (void *)cache_resource,
 832                                 rte_atomic32_read(&cache_resource->refcnt));
 833                         rte_atomic32_inc(&cache_resource->refcnt);
 834                         dev_flow->dv.encap_decap = cache_resource;
 835                         return 0;
 836                 }
 837         }
 838         /* Register new encap/decap resource. */
 839         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
 840         if (!cache_resource)
 841                 return rte_flow_error_set(error, ENOMEM,
 842                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 843                                           "cannot allocate resource memory");
 844         *cache_resource = *resource;
 845         cache_resource->verbs_action =
 846                 mlx5_glue->dv_create_flow_action_packet_reformat
 847                         (priv->sh->ctx, cache_resource->reformat_type,
 848                          cache_resource->ft_type, ns, cache_resource->flags,
 849                          cache_resource->size,
 850                          (cache_resource->size ? cache_resource->buf : NULL));
 851         if (!cache_resource->verbs_action) {
 852                 rte_free(cache_resource);
 853                 return rte_flow_error_set(error, ENOMEM,
 854                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 855                                           NULL, "cannot create action");
 856         }
 857         rte_atomic32_init(&cache_resource->refcnt);
 858         rte_atomic32_inc(&cache_resource->refcnt);
 859         LIST_INSERT_HEAD(&priv->encaps_decaps, cache_resource, next);
 860         dev_flow->dv.encap_decap = cache_resource;
 861         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
 862                 (void *)cache_resource,
 863                 rte_atomic32_read(&cache_resource->refcnt));
 864         return 0;
 865 }
 866
 867 /**
 868  * Get the size of specific rte_flow_item_type
 869  *
 870  * @param[in] item_type
 871  *   Tested rte_flow_item_type.
 872  *
 873  * @return
 874  *   sizeof struct item_type, 0 if void or irrelevant.
 875  */
 876 static size_t
 877 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
 878 {
 879         size_t retval;
 880
 881         switch (item_type) {
 882         case RTE_FLOW_ITEM_TYPE_ETH:
 883                 retval = sizeof(struct rte_flow_item_eth);
 884                 break;
 885         case RTE_FLOW_ITEM_TYPE_VLAN:
 886                 retval = sizeof(struct rte_flow_item_vlan);
 887                 break;
 888         case RTE_FLOW_ITEM_TYPE_IPV4:
 889                 retval = sizeof(struct rte_flow_item_ipv4);
 890                 break;
 891         case RTE_FLOW_ITEM_TYPE_IPV6:
 892                 retval = sizeof(struct rte_flow_item_ipv6);
 893                 break;
 894         case RTE_FLOW_ITEM_TYPE_UDP:
 895                 retval = sizeof(struct rte_flow_item_udp);
 896                 break;
 897         case RTE_FLOW_ITEM_TYPE_TCP:
 898                 retval = sizeof(struct rte_flow_item_tcp);
 899                 break;
 900         case RTE_FLOW_ITEM_TYPE_VXLAN:
 901                 retval = sizeof(struct rte_flow_item_vxlan);
 902                 break;
 903         case RTE_FLOW_ITEM_TYPE_GRE:
 904                 retval = sizeof(struct rte_flow_item_gre);
 905                 break;
 906         case RTE_FLOW_ITEM_TYPE_NVGRE:
 907                 retval = sizeof(struct rte_flow_item_nvgre);
 908                 break;
 909         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 910                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
 911                 break;
 912         case RTE_FLOW_ITEM_TYPE_MPLS:
 913                 retval = sizeof(struct rte_flow_item_mpls);
 914                 break;
 915         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
 916         default:
 917                 retval = 0;
 918                 break;
 919         }
 920         return retval;
 921 }
 922
 923 #define MLX5_ENCAP_IPV4_VERSION         0x40
 924 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
 925 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
 926 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
 927 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
 928 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
 929 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
 930
 931 /**
 932  * Convert the encap action data from list of rte_flow_item to raw buffer
 933  *
 934  * @param[in] items
 935  *   Pointer to rte_flow_item objects list.
 936  * @param[out] buf
 937  *   Pointer to the output buffer.
 938  * @param[out] size
 939  *   Pointer to the output buffer size.
 940  * @param[out] error
 941  *   Pointer to the error structure.
 942  *
 943  * @return
 944  *   0 on success, a negative errno value otherwise and rte_errno is set.
 945  */
 946 static int
 947 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 948                            size_t *size, struct rte_flow_error *error)
 949 {
 950         struct ether_hdr *eth = NULL;
 951         struct vlan_hdr *vlan = NULL;
 952         struct ipv4_hdr *ipv4 = NULL;
 953         struct ipv6_hdr *ipv6 = NULL;
 954         struct udp_hdr *udp = NULL;
 955         struct vxlan_hdr *vxlan = NULL;
 956         struct vxlan_gpe_hdr *vxlan_gpe = NULL;
 957         struct gre_hdr *gre = NULL;
 958         size_t len;
 959         size_t temp_size = 0;
 960
 961         if (!items)
 962                 return rte_flow_error_set(error, EINVAL,
 963                                           RTE_FLOW_ERROR_TYPE_ACTION,
 964                                           NULL, "invalid empty data");
 965         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 966                 len = flow_dv_get_item_len(items->type);
 967                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
 968                         return rte_flow_error_set(error, EINVAL,
 969                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 970                                                   (void *)items->type,
 971                                                   "items total size is too big"
 972                                                   " for encap action");
 973                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
 974                 switch (items->type) {
 975                 case RTE_FLOW_ITEM_TYPE_ETH:
 976                         eth = (struct ether_hdr *)&buf[temp_size];
 977                         break;
 978                 case RTE_FLOW_ITEM_TYPE_VLAN:
 979                         vlan = (struct vlan_hdr *)&buf[temp_size];
 980                         if (!eth)
 981                                 return rte_flow_error_set(error, EINVAL,
 982                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 983                                                 (void *)items->type,
 984                                                 "eth header not found");
 985                         if (!eth->ether_type)
 986                                 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
 987                         break;
 988                 case RTE_FLOW_ITEM_TYPE_IPV4:
 989                         ipv4 = (struct ipv4_hdr *)&buf[temp_size];
 990                         if (!vlan && !eth)
 991                                 return rte_flow_error_set(error, EINVAL,
 992                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 993                                                 (void *)items->type,
 994                                                 "neither eth nor vlan"
 995                                                 " header found");
 996                         if (vlan && !vlan->eth_proto)
 997                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
 998                         else if (eth && !eth->ether_type)
 999                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
1000                         if (!ipv4->version_ihl)
1001                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1002                                                     MLX5_ENCAP_IPV4_IHL_MIN;
1003                         if (!ipv4->time_to_live)
1004                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1005                         break;
1006                 case RTE_FLOW_ITEM_TYPE_IPV6:
1007                         ipv6 = (struct ipv6_hdr *)&buf[temp_size];
1008                         if (!vlan && !eth)
1009                                 return rte_flow_error_set(error, EINVAL,
1010                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1011                                                 (void *)items->type,
1012                                                 "neither eth nor vlan"
1013                                                 " header found");
1014                         if (vlan && !vlan->eth_proto)
1015                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
1016                         else if (eth && !eth->ether_type)
1017                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
1018                         if (!ipv6->vtc_flow)
1019                                 ipv6->vtc_flow =
1020                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1021                         if (!ipv6->hop_limits)
1022                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1023                         break;
1024                 case RTE_FLOW_ITEM_TYPE_UDP:
1025                         udp = (struct udp_hdr *)&buf[temp_size];
1026                         if (!ipv4 && !ipv6)
1027                                 return rte_flow_error_set(error, EINVAL,
1028                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1029                                                 (void *)items->type,
1030                                                 "ip header not found");
1031                         if (ipv4 && !ipv4->next_proto_id)
1032                                 ipv4->next_proto_id = IPPROTO_UDP;
1033                         else if (ipv6 && !ipv6->proto)
1034                                 ipv6->proto = IPPROTO_UDP;
1035                         break;
1036                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1037                         vxlan = (struct vxlan_hdr *)&buf[temp_size];
1038                         if (!udp)
1039                                 return rte_flow_error_set(error, EINVAL,
1040                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1041                                                 (void *)items->type,
1042                                                 "udp header not found");
1043                         if (!udp->dst_port)
1044                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1045                         if (!vxlan->vx_flags)
1046                                 vxlan->vx_flags =
1047                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1048                         break;
1049                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1050                         vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
1051                         if (!udp)
1052                                 return rte_flow_error_set(error, EINVAL,
1053                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1054                                                 (void *)items->type,
1055                                                 "udp header not found");
1056                         if (!vxlan_gpe->proto)
1057                                 return rte_flow_error_set(error, EINVAL,
1058                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1059                                                 (void *)items->type,
1060                                                 "next protocol not found");
1061                         if (!udp->dst_port)
1062                                 udp->dst_port =
1063                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1064                         if (!vxlan_gpe->vx_flags)
1065                                 vxlan_gpe->vx_flags =
1066                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1067                         break;
1068                 case RTE_FLOW_ITEM_TYPE_GRE:
1069                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1070                         gre = (struct gre_hdr *)&buf[temp_size];
1071                         if (!gre->proto)
1072                                 return rte_flow_error_set(error, EINVAL,
1073                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1074                                                 (void *)items->type,
1075                                                 "next protocol not found");
1076                         if (!ipv4 && !ipv6)
1077                                 return rte_flow_error_set(error, EINVAL,
1078                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1079                                                 (void *)items->type,
1080                                                 "ip header not found");
1081                         if (ipv4 && !ipv4->next_proto_id)
1082                                 ipv4->next_proto_id = IPPROTO_GRE;
1083                         else if (ipv6 && !ipv6->proto)
1084                                 ipv6->proto = IPPROTO_GRE;
1085                         break;
1086                 case RTE_FLOW_ITEM_TYPE_VOID:
1087                         break;
1088                 default:
1089                         return rte_flow_error_set(error, EINVAL,
1090                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1091                                                   (void *)items->type,
1092                                                   "unsupported item type");
1093                         break;
1094                 }
1095                 temp_size += len;
1096         }
1097         *size = temp_size;
1098         return 0;
1099 }
1100
1101 /**
1102  * Convert L2 encap action to DV specification.
1103  *
1104  * @param[in] dev
1105  *   Pointer to rte_eth_dev structure.
1106  * @param[in] action
1107  *   Pointer to action structure.
1108  * @param[in, out] dev_flow
1109  *   Pointer to the mlx5_flow.
1110  * @param[out] error
1111  *   Pointer to the error structure.
1112  *
1113  * @return
1114  *   0 on success, a negative errno value otherwise and rte_errno is set.
1115  */
1116 static int
1117 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1118                                const struct rte_flow_action *action,
1119                                struct mlx5_flow *dev_flow,
1120                                struct rte_flow_error *error)
1121 {
1122         const struct rte_flow_item *encap_data;
1123         const struct rte_flow_action_raw_encap *raw_encap_data;
1124         struct mlx5_flow_dv_encap_decap_resource res = {
1125                 .reformat_type =
1126                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1127                 .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1128         };
1129
1130         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1131                 raw_encap_data =
1132                         (const struct rte_flow_action_raw_encap *)action->conf;
1133                 res.size = raw_encap_data->size;
1134                 memcpy(res.buf, raw_encap_data->data, res.size);
1135         } else {
1136                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1137                         encap_data =
1138                                 ((const struct rte_flow_action_vxlan_encap *)
1139                                                 action->conf)->definition;
1140                 else
1141                         encap_data =
1142                                 ((const struct rte_flow_action_nvgre_encap *)
1143                                                 action->conf)->definition;
1144                 if (flow_dv_convert_encap_data(encap_data, res.buf,
1145                                                &res.size, error))
1146                         return -rte_errno;
1147         }
1148         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1149                 return rte_flow_error_set(error, EINVAL,
1150                                           RTE_FLOW_ERROR_TYPE_ACTION,
1151                                           NULL, "can't create L2 encap action");
1152         return 0;
1153 }
1154
1155 /**
1156  * Convert L2 decap action to DV specification.
1157  *
1158  * @param[in] dev
1159  *   Pointer to rte_eth_dev structure.
1160  * @param[in, out] dev_flow
1161  *   Pointer to the mlx5_flow.
1162  * @param[out] error
1163  *   Pointer to the error structure.
1164  *
1165  * @return
1166  *   0 on success, a negative errno value otherwise and rte_errno is set.
1167  */
1168 static int
1169 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1170                                struct mlx5_flow *dev_flow,
1171                                struct rte_flow_error *error)
1172 {
1173         struct mlx5_flow_dv_encap_decap_resource res = {
1174                 .size = 0,
1175                 .reformat_type =
1176                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1177                 .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1178         };
1179
1180         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1181                 return rte_flow_error_set(error, EINVAL,
1182                                           RTE_FLOW_ERROR_TYPE_ACTION,
1183                                           NULL, "can't create L2 decap action");
1184         return 0;
1185 }
1186
1187 /**
1188  * Convert raw decap/encap (L3 tunnel) action to DV specification.
1189  *
1190  * @param[in] dev
1191  *   Pointer to rte_eth_dev structure.
1192  * @param[in] action
1193  *   Pointer to action structure.
1194  * @param[in, out] dev_flow
1195  *   Pointer to the mlx5_flow.
1196  * @param[in] attr
1197  *   Pointer to the flow attributes.
1198  * @param[out] error
1199  *   Pointer to the error structure.
1200  *
1201  * @return
1202  *   0 on success, a negative errno value otherwise and rte_errno is set.
1203  */
1204 static int
1205 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1206                                 const struct rte_flow_action *action,
1207                                 struct mlx5_flow *dev_flow,
1208                                 const struct rte_flow_attr *attr,
1209                                 struct rte_flow_error *error)
1210 {
1211         const struct rte_flow_action_raw_encap *encap_data;
1212         struct mlx5_flow_dv_encap_decap_resource res;
1213
1214         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1215         res.size = encap_data->size;
1216         memcpy(res.buf, encap_data->data, res.size);
1217         res.reformat_type = attr->egress ?
1218                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1219                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1220         res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1221                                      MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1222         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1223                 return rte_flow_error_set(error, EINVAL,
1224                                           RTE_FLOW_ERROR_TYPE_ACTION,
1225                                           NULL, "can't create encap action");
1226         return 0;
1227 }
1228
1229 /**
1230  * Validate the modify-header actions.
1231  *
1232  * @param[in] action_flags
1233  *   Holds the actions detected until now.
1234  * @param[in] action
1235  *   Pointer to the modify action.
1236  * @param[out] error
1237  *   Pointer to error structure.
1238  *
1239  * @return
1240  *   0 on success, a negative errno value otherwise and rte_errno is set.
1241  */
1242 static int
1243 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1244                                    const struct rte_flow_action *action,
1245                                    struct rte_flow_error *error)
1246 {
1247         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1248                 return rte_flow_error_set(error, EINVAL,
1249                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1250                                           NULL, "action configuration not set");
1251         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1252                 return rte_flow_error_set(error, EINVAL,
1253                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1254                                           "can't have encap action before"
1255                                           " modify action");
1256         return 0;
1257 }
1258
1259 /**
1260  * Validate the modify-header MAC address actions.
1261  *
1262  * @param[in] action_flags
1263  *   Holds the actions detected until now.
1264  * @param[in] action
1265  *   Pointer to the modify action.
1266  * @param[in] item_flags
1267  *   Holds the items detected.
1268  * @param[out] error
1269  *   Pointer to error structure.
1270  *
1271  * @return
1272  *   0 on success, a negative errno value otherwise and rte_errno is set.
1273  */
1274 static int
1275 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1276                                    const struct rte_flow_action *action,
1277                                    const uint64_t item_flags,
1278                                    struct rte_flow_error *error)
1279 {
1280         int ret = 0;
1281
1282         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1283         if (!ret) {
1284                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1285                         return rte_flow_error_set(error, EINVAL,
1286                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1287                                                   NULL,
1288                                                   "no L2 item in pattern");
1289         }
1290         return ret;
1291 }
1292
1293 /**
1294  * Validate the modify-header IPv4 address actions.
1295  *
1296  * @param[in] action_flags
1297  *   Holds the actions detected until now.
1298  * @param[in] action
1299  *   Pointer to the modify action.
1300  * @param[in] item_flags
1301  *   Holds the items detected.
1302  * @param[out] error
1303  *   Pointer to error structure.
1304  *
1305  * @return
1306  *   0 on success, a negative errno value otherwise and rte_errno is set.
1307  */
1308 static int
1309 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
1310                                     const struct rte_flow_action *action,
1311                                     const uint64_t item_flags,
1312                                     struct rte_flow_error *error)
1313 {
1314         int ret = 0;
1315
1316         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1317         if (!ret) {
1318                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
1319                         return rte_flow_error_set(error, EINVAL,
1320                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1321                                                   NULL,
1322                                                   "no ipv4 item in pattern");
1323         }
1324         return ret;
1325 }
1326
1327 /**
1328  * Validate the modify-header IPv6 address actions.
1329  *
1330  * @param[in] action_flags
1331  *   Holds the actions detected until now.
1332  * @param[in] action
1333  *   Pointer to the modify action.
1334  * @param[in] item_flags
1335  *   Holds the items detected.
1336  * @param[out] error
1337  *   Pointer to error structure.
1338  *
1339  * @return
1340  *   0 on success, a negative errno value otherwise and rte_errno is set.
1341  */
1342 static int
1343 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
1344                                     const struct rte_flow_action *action,
1345                                     const uint64_t item_flags,
1346                                     struct rte_flow_error *error)
1347 {
1348         int ret = 0;
1349
1350         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1351         if (!ret) {
1352                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
1353                         return rte_flow_error_set(error, EINVAL,
1354                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1355                                                   NULL,
1356                                                   "no ipv6 item in pattern");
1357         }
1358         return ret;
1359 }
1360
1361 /**
1362  * Validate the modify-header TP actions.
1363  *
1364  * @param[in] action_flags
1365  *   Holds the actions detected until now.
1366  * @param[in] action
1367  *   Pointer to the modify action.
1368  * @param[in] item_flags
1369  *   Holds the items detected.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
1378                                   const struct rte_flow_action *action,
1379                                   const uint64_t item_flags,
1380                                   struct rte_flow_error *error)
1381 {
1382         int ret = 0;
1383
1384         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1385         if (!ret) {
1386                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
1387                         return rte_flow_error_set(error, EINVAL,
1388                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1389                                                   NULL, "no transport layer "
1390                                                   "in pattern");
1391         }
1392         return ret;
1393 }
1394
1395 /**
1396  * Validate the modify-header TTL actions.
1397  *
1398  * @param[in] action_flags
1399  *   Holds the actions detected until now.
1400  * @param[in] action
1401  *   Pointer to the modify action.
1402  * @param[in] item_flags
1403  *   Holds the items detected.
1404  * @param[out] error
1405  *   Pointer to error structure.
1406  *
1407  * @return
1408  *   0 on success, a negative errno value otherwise and rte_errno is set.
1409  */
1410 static int
1411 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
1412                                    const struct rte_flow_action *action,
1413                                    const uint64_t item_flags,
1414                                    struct rte_flow_error *error)
1415 {
1416         int ret = 0;
1417
1418         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1419         if (!ret) {
1420                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
1421                         return rte_flow_error_set(error, EINVAL,
1422                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1423                                                   NULL,
1424                                                   "no IP protocol in pattern");
1425         }
1426         return ret;
1427 }
1428
1429 /**
1430  * Find existing modify-header resource or create and register a new one.
1431  *
1432  * @param dev[in, out]
1433  *   Pointer to rte_eth_dev structure.
1434  * @param[in, out] resource
1435  *   Pointer to modify-header resource.
1436  * @parm[in, out] dev_flow
1437  *   Pointer to the dev_flow.
1438  * @param[out] error
1439  *   pointer to error structure.
1440  *
1441  * @return
1442  *   0 on success otherwise -errno and errno is set.
1443  */
1444 static int
1445 flow_dv_modify_hdr_resource_register
1446                         (struct rte_eth_dev *dev,
1447                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1448                          struct mlx5_flow *dev_flow,
1449                          struct rte_flow_error *error)
1450 {
1451         struct mlx5_priv *priv = dev->data->dev_private;
1452         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
1453
1454         struct mlx5dv_dr_ns *ns =
1455                 resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX  ?
1456                 priv->tx_ns : priv->rx_ns;
1457
1458         /* Lookup a matching resource from cache. */
1459         LIST_FOREACH(cache_resource, &priv->modify_cmds, next) {
1460                 if (resource->ft_type == cache_resource->ft_type &&
1461                     resource->actions_num == cache_resource->actions_num &&
1462                     !memcmp((const void *)resource->actions,
1463                             (const void *)cache_resource->actions,
1464                             (resource->actions_num *
1465                                             sizeof(resource->actions[0])))) {
1466                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
1467                                 (void *)cache_resource,
1468                                 rte_atomic32_read(&cache_resource->refcnt));
1469                         rte_atomic32_inc(&cache_resource->refcnt);
1470                         dev_flow->dv.modify_hdr = cache_resource;
1471                         return 0;
1472                 }
1473         }
1474         /* Register new modify-header resource. */
1475         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1476         if (!cache_resource)
1477                 return rte_flow_error_set(error, ENOMEM,
1478                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1479                                           "cannot allocate resource memory");
1480         *cache_resource = *resource;
1481         cache_resource->verbs_action =
1482                 mlx5_glue->dv_create_flow_action_modify_header
1483                                         (priv->sh->ctx, cache_resource->ft_type,
1484                                          ns, 0,
1485                                          cache_resource->actions_num *
1486                                          sizeof(cache_resource->actions[0]),
1487                                          (uint64_t *)cache_resource->actions);
1488         if (!cache_resource->verbs_action) {
1489                 rte_free(cache_resource);
1490                 return rte_flow_error_set(error, ENOMEM,
1491                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1492                                           NULL, "cannot create action");
1493         }
1494         rte_atomic32_init(&cache_resource->refcnt);
1495         rte_atomic32_inc(&cache_resource->refcnt);
1496         LIST_INSERT_HEAD(&priv->modify_cmds, cache_resource, next);
1497         dev_flow->dv.modify_hdr = cache_resource;
1498         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
1499                 (void *)cache_resource,
1500                 rte_atomic32_read(&cache_resource->refcnt));
1501         return 0;
1502 }
1503
1504 /**
1505  * Get or create a flow counter.
1506  *
1507  * @param[in] dev
1508  *   Pointer to the Ethernet device structure.
1509  * @param[in] shared
1510  *   Indicate if this counter is shared with other flows.
1511  * @param[in] id
1512  *   Counter identifier.
1513  *
1514  * @return
1515  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
1516  */
1517 static struct mlx5_flow_counter *
1518 flow_dv_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
1519 {
1520         struct mlx5_priv *priv = dev->data->dev_private;
1521         struct mlx5_flow_counter *cnt = NULL;
1522         struct mlx5_devx_counter_set *dcs = NULL;
1523         int ret;
1524
1525         if (!priv->config.devx) {
1526                 ret = -ENOTSUP;
1527                 goto error_exit;
1528         }
1529         if (shared) {
1530                 LIST_FOREACH(cnt, &priv->flow_counters, next) {
1531                         if (cnt->shared && cnt->id == id) {
1532                                 cnt->ref_cnt++;
1533                                 return cnt;
1534                         }
1535                 }
1536         }
1537         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
1538         dcs = rte_calloc(__func__, 1, sizeof(*dcs), 0);
1539         if (!dcs || !cnt) {
1540                 ret = -ENOMEM;
1541                 goto error_exit;
1542         }
1543         ret = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, dcs);
1544         if (ret)
1545                 goto error_exit;
1546         struct mlx5_flow_counter tmpl = {
1547                 .shared = shared,
1548                 .ref_cnt = 1,
1549                 .id = id,
1550                 .dcs = dcs,
1551         };
1552         tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
1553         if (!tmpl.action) {
1554                 ret = errno;
1555                 goto error_exit;
1556         }
1557         *cnt = tmpl;
1558         LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
1559         return cnt;
1560 error_exit:
1561         rte_free(cnt);
1562         rte_free(dcs);
1563         rte_errno = -ret;
1564         return NULL;
1565 }
1566
1567 /**
1568  * Release a flow counter.
1569  *
1570  * @param[in] counter
1571  *   Pointer to the counter handler.
1572  */
1573 static void
1574 flow_dv_counter_release(struct mlx5_flow_counter *counter)
1575 {
1576         int ret;
1577
1578         if (!counter)
1579                 return;
1580         if (--counter->ref_cnt == 0) {
1581                 ret = mlx5_devx_cmd_flow_counter_free(counter->dcs->obj);
1582                 if (ret)
1583                         DRV_LOG(ERR, "Failed to free devx counters, %d", ret);
1584                 LIST_REMOVE(counter, next);
1585                 rte_free(counter->dcs);
1586                 rte_free(counter);
1587         }
1588 }
1589
1590 /**
1591  * Verify the @p attributes will be correctly understood by the NIC and store
1592  * them in the @p flow if everything is correct.
1593  *
1594  * @param[in] dev
1595  *   Pointer to dev struct.
1596  * @param[in] attributes
1597  *   Pointer to flow attributes
1598  * @param[out] error
1599  *   Pointer to error structure.
1600  *
1601  * @return
1602  *   0 on success, a negative errno value otherwise and rte_errno is set.
1603  */
1604 static int
1605 flow_dv_validate_attributes(struct rte_eth_dev *dev,
1606                             const struct rte_flow_attr *attributes,
1607                             struct rte_flow_error *error)
1608 {
1609         struct mlx5_priv *priv = dev->data->dev_private;
1610         uint32_t priority_max = priv->config.flow_prio - 1;
1611
1612 #ifdef HAVE_MLX5DV_DR
1613         if (attributes->group)
1614                 return rte_flow_error_set(error, ENOTSUP,
1615                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1616                                           NULL,
1617                                           "groups is not supported");
1618 #endif
1619         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1620             attributes->priority >= priority_max)
1621                 return rte_flow_error_set(error, ENOTSUP,
1622                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1623                                           NULL,
1624                                           "priority out of range");
1625         if (attributes->transfer)
1626                 return rte_flow_error_set(error, ENOTSUP,
1627                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1628                                           NULL,
1629                                           "transfer is not supported");
1630         if (!(attributes->egress ^ attributes->ingress))
1631                 return rte_flow_error_set(error, ENOTSUP,
1632                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
1633                                           "must specify exactly one of "
1634                                           "ingress or egress");
1635         return 0;
1636 }
1637
1638 /**
1639  * Internal validation function. For validating both actions and items.
1640  *
1641  * @param[in] dev
1642  *   Pointer to the rte_eth_dev structure.
1643  * @param[in] attr
1644  *   Pointer to the flow attributes.
1645  * @param[in] items
1646  *   Pointer to the list of items.
1647  * @param[in] actions
1648  *   Pointer to the list of actions.
1649  * @param[out] error
1650  *   Pointer to the error structure.
1651  *
1652  * @return
1653  *   0 on success, a negative errno value otherwise and rte_errno is set.
1654  */
1655 static int
1656 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
1657                  const struct rte_flow_item items[],
1658                  const struct rte_flow_action actions[],
1659                  struct rte_flow_error *error)
1660 {
1661         int ret;
1662         uint64_t action_flags = 0;
1663         uint64_t item_flags = 0;
1664         uint64_t last_item = 0;
1665         int tunnel = 0;
1666         uint8_t next_protocol = 0xff;
1667         int actions_n = 0;
1668
1669         if (items == NULL)
1670                 return -1;
1671         ret = flow_dv_validate_attributes(dev, attr, error);
1672         if (ret < 0)
1673                 return ret;
1674         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1675                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1676                 switch (items->type) {
1677                 case RTE_FLOW_ITEM_TYPE_VOID:
1678                         break;
1679                 case RTE_FLOW_ITEM_TYPE_ETH:
1680                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1681                                                           error);
1682                         if (ret < 0)
1683                                 return ret;
1684                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1685                                              MLX5_FLOW_LAYER_OUTER_L2;
1686                         break;
1687                 case RTE_FLOW_ITEM_TYPE_VLAN:
1688                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1689                                                            error);
1690                         if (ret < 0)
1691                                 return ret;
1692                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1693                                              MLX5_FLOW_LAYER_OUTER_VLAN;
1694                         break;
1695                 case RTE_FLOW_ITEM_TYPE_IPV4:
1696                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1697                                                            NULL, error);
1698                         if (ret < 0)
1699                                 return ret;
1700                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1701                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1702                         if (items->mask != NULL &&
1703                             ((const struct rte_flow_item_ipv4 *)
1704                              items->mask)->hdr.next_proto_id) {
1705                                 next_protocol =
1706                                         ((const struct rte_flow_item_ipv4 *)
1707                                          (items->spec))->hdr.next_proto_id;
1708                                 next_protocol &=
1709                                         ((const struct rte_flow_item_ipv4 *)
1710                                          (items->mask))->hdr.next_proto_id;
1711                         } else {
1712                                 /* Reset for inner layer. */
1713                                 next_protocol = 0xff;
1714                         }
1715                         break;
1716                 case RTE_FLOW_ITEM_TYPE_IPV6:
1717                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1718                                                            NULL, error);
1719                         if (ret < 0)
1720                                 return ret;
1721                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1722                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1723                         if (items->mask != NULL &&
1724                             ((const struct rte_flow_item_ipv6 *)
1725                              items->mask)->hdr.proto) {
1726                                 next_protocol =
1727                                         ((const struct rte_flow_item_ipv6 *)
1728                                          items->spec)->hdr.proto;
1729                                 next_protocol &=
1730                                         ((const struct rte_flow_item_ipv6 *)
1731                                          items->mask)->hdr.proto;
1732                         } else {
1733                                 /* Reset for inner layer. */
1734                                 next_protocol = 0xff;
1735                         }
1736                         break;
1737                 case RTE_FLOW_ITEM_TYPE_TCP:
1738                         ret = mlx5_flow_validate_item_tcp
1739                                                 (items, item_flags,
1740                                                  next_protocol,
1741                                                  &rte_flow_item_tcp_mask,
1742                                                  error);
1743                         if (ret < 0)
1744                                 return ret;
1745                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1746                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1747                         break;
1748                 case RTE_FLOW_ITEM_TYPE_UDP:
1749                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1750                                                           next_protocol,
1751                                                           error);
1752                         if (ret < 0)
1753                                 return ret;
1754                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1755                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1756                         break;
1757                 case RTE_FLOW_ITEM_TYPE_GRE:
1758                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1759                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1760                                                           next_protocol, error);
1761                         if (ret < 0)
1762                                 return ret;
1763                         last_item = MLX5_FLOW_LAYER_GRE;
1764                         break;
1765                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1766                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1767                                                             error);
1768                         if (ret < 0)
1769                                 return ret;
1770                         last_item = MLX5_FLOW_LAYER_VXLAN;
1771                         break;
1772                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1773                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1774                                                                 item_flags, dev,
1775                                                                 error);
1776                         if (ret < 0)
1777                                 return ret;
1778                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1779                         break;
1780                 case RTE_FLOW_ITEM_TYPE_MPLS:
1781                         ret = mlx5_flow_validate_item_mpls(dev, items,
1782                                                            item_flags,
1783                                                            last_item, error);
1784                         if (ret < 0)
1785                                 return ret;
1786                         last_item = MLX5_FLOW_LAYER_MPLS;
1787                         break;
1788                 case RTE_FLOW_ITEM_TYPE_META:
1789                         ret = flow_dv_validate_item_meta(dev, items, attr,
1790                                                          error);
1791                         if (ret < 0)
1792                                 return ret;
1793                         last_item = MLX5_FLOW_ITEM_METADATA;
1794                         break;
1795                 default:
1796                         return rte_flow_error_set(error, ENOTSUP,
1797                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1798                                                   NULL, "item not supported");
1799                 }
1800                 item_flags |= last_item;
1801         }
1802         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1803                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
1804                         return rte_flow_error_set(error, ENOTSUP,
1805                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1806                                                   actions, "too many actions");
1807                 switch (actions->type) {
1808                 case RTE_FLOW_ACTION_TYPE_VOID:
1809                         break;
1810                 case RTE_FLOW_ACTION_TYPE_FLAG:
1811                         ret = mlx5_flow_validate_action_flag(action_flags,
1812                                                              attr, error);
1813                         if (ret < 0)
1814                                 return ret;
1815                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1816                         ++actions_n;
1817                         break;
1818                 case RTE_FLOW_ACTION_TYPE_MARK:
1819                         ret = mlx5_flow_validate_action_mark(actions,
1820                                                              action_flags,
1821                                                              attr, error);
1822                         if (ret < 0)
1823                                 return ret;
1824                         action_flags |= MLX5_FLOW_ACTION_MARK;
1825                         ++actions_n;
1826                         break;
1827                 case RTE_FLOW_ACTION_TYPE_DROP:
1828                         ret = mlx5_flow_validate_action_drop(action_flags,
1829                                                              attr, error);
1830                         if (ret < 0)
1831                                 return ret;
1832                         action_flags |= MLX5_FLOW_ACTION_DROP;
1833                         ++actions_n;
1834                         break;
1835                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1836                         ret = mlx5_flow_validate_action_queue(actions,
1837                                                               action_flags, dev,
1838                                                               attr, error);
1839                         if (ret < 0)
1840                                 return ret;
1841                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1842                         ++actions_n;
1843                         break;
1844                 case RTE_FLOW_ACTION_TYPE_RSS:
1845                         ret = mlx5_flow_validate_action_rss(actions,
1846                                                             action_flags, dev,
1847                                                             attr, error);
1848                         if (ret < 0)
1849                                 return ret;
1850                         action_flags |= MLX5_FLOW_ACTION_RSS;
1851                         ++actions_n;
1852                         break;
1853                 case RTE_FLOW_ACTION_TYPE_COUNT:
1854                         ret = flow_dv_validate_action_count(dev, error);
1855                         if (ret < 0)
1856                                 return ret;
1857                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1858                         ++actions_n;
1859                         break;
1860                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1861                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
1862                         ret = flow_dv_validate_action_l2_encap(action_flags,
1863                                                                actions, attr,
1864                                                                error);
1865                         if (ret < 0)
1866                                 return ret;
1867                         action_flags |= actions->type ==
1868                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
1869                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
1870                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
1871                         ++actions_n;
1872                         break;
1873                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1874                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
1875                         ret = flow_dv_validate_action_l2_decap(action_flags,
1876                                                                attr, error);
1877                         if (ret < 0)
1878                                 return ret;
1879                         action_flags |= actions->type ==
1880                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
1881                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
1882                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
1883                         ++actions_n;
1884                         break;
1885                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
1886                         ret = flow_dv_validate_action_raw_encap(action_flags,
1887                                                                 actions, attr,
1888                                                                 error);
1889                         if (ret < 0)
1890                                 return ret;
1891                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
1892                         ++actions_n;
1893                         break;
1894                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
1895                         ret = flow_dv_validate_action_raw_decap(action_flags,
1896                                                                 actions, attr,
1897                                                                 error);
1898                         if (ret < 0)
1899                                 return ret;
1900                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
1901                         ++actions_n;
1902                         break;
1903                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1904                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1905                         ret = flow_dv_validate_action_modify_mac(action_flags,
1906                                                                  actions,
1907                                                                  item_flags,
1908                                                                  error);
1909                         if (ret < 0)
1910                                 return ret;
1911                         /* Count all modify-header actions as one action. */
1912                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1913                                 ++actions_n;
1914                         action_flags |= actions->type ==
1915                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
1916                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
1917                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
1918                         break;
1919
1920                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1921                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1922                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
1923                                                                   actions,
1924                                                                   item_flags,
1925                                                                   error);
1926                         if (ret < 0)
1927                                 return ret;
1928                         /* Count all modify-header actions as one action. */
1929                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1930                                 ++actions_n;
1931                         action_flags |= actions->type ==
1932                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
1933                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
1934                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
1935                         break;
1936                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1937                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1938                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
1939                                                                   actions,
1940                                                                   item_flags,
1941                                                                   error);
1942                         if (ret < 0)
1943                                 return ret;
1944                         /* Count all modify-header actions as one action. */
1945                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1946                                 ++actions_n;
1947                         action_flags |= actions->type ==
1948                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
1949                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
1950                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
1951                         break;
1952                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1953                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1954                         ret = flow_dv_validate_action_modify_tp(action_flags,
1955                                                                 actions,
1956                                                                 item_flags,
1957                                                                 error);
1958                         if (ret < 0)
1959                                 return ret;
1960                         /* Count all modify-header actions as one action. */
1961                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1962                                 ++actions_n;
1963                         action_flags |= actions->type ==
1964                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
1965                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
1966                                                 MLX5_FLOW_ACTION_SET_TP_DST;
1967                         break;
1968                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1969                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1970                         ret = flow_dv_validate_action_modify_ttl(action_flags,
1971                                                                  actions,
1972                                                                  item_flags,
1973                                                                  error);
1974                         if (ret < 0)
1975                                 return ret;
1976                         /* Count all modify-header actions as one action. */
1977                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1978                                 ++actions_n;
1979                         action_flags |= actions->type ==
1980                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
1981                                                 MLX5_FLOW_ACTION_SET_TTL :
1982                                                 MLX5_FLOW_ACTION_DEC_TTL;
1983                         break;
1984                 default:
1985                         return rte_flow_error_set(error, ENOTSUP,
1986                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1987                                                   actions,
1988                                                   "action not supported");
1989                 }
1990         }
1991         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
1992                 return rte_flow_error_set(error, EINVAL,
1993                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1994                                           "no fate action is found");
1995         return 0;
1996 }
1997
1998 /**
1999  * Internal preparation function. Allocates the DV flow size,
2000  * this size is constant.
2001  *
2002  * @param[in] attr
2003  *   Pointer to the flow attributes.
2004  * @param[in] items
2005  *   Pointer to the list of items.
2006  * @param[in] actions
2007  *   Pointer to the list of actions.
2008  * @param[out] error
2009  *   Pointer to the error structure.
2010  *
2011  * @return
2012  *   Pointer to mlx5_flow object on success,
2013  *   otherwise NULL and rte_errno is set.
2014  */
2015 static struct mlx5_flow *
2016 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
2017                 const struct rte_flow_item items[] __rte_unused,
2018                 const struct rte_flow_action actions[] __rte_unused,
2019                 struct rte_flow_error *error)
2020 {
2021         uint32_t size = sizeof(struct mlx5_flow);
2022         struct mlx5_flow *flow;
2023
2024         flow = rte_calloc(__func__, 1, size, 0);
2025         if (!flow) {
2026                 rte_flow_error_set(error, ENOMEM,
2027                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2028                                    "not enough memory to create flow");
2029                 return NULL;
2030         }
2031         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
2032         return flow;
2033 }
2034
2035 #ifndef NDEBUG
2036 /**
2037  * Sanity check for match mask and value. Similar to check_valid_spec() in
2038  * kernel driver. If unmasked bit is present in value, it returns failure.
2039  *
2040  * @param match_mask
2041  *   pointer to match mask buffer.
2042  * @param match_value
2043  *   pointer to match value buffer.
2044  *
2045  * @return
2046  *   0 if valid, -EINVAL otherwise.
2047  */
2048 static int
2049 flow_dv_check_valid_spec(void *match_mask, void *match_value)
2050 {
2051         uint8_t *m = match_mask;
2052         uint8_t *v = match_value;
2053         unsigned int i;
2054
2055         for (i = 0; i < MLX5_ST_SZ_DB(fte_match_param); ++i) {
2056                 if (v[i] & ~m[i]) {
2057                         DRV_LOG(ERR,
2058                                 "match_value differs from match_criteria"
2059                                 " %p[%u] != %p[%u]",
2060                                 match_value, i, match_mask, i);
2061                         return -EINVAL;
2062                 }
2063         }
2064         return 0;
2065 }
2066 #endif
2067
2068 /**
2069  * Add Ethernet item to matcher and to the value.
2070  *
2071  * @param[in, out] matcher
2072  *   Flow matcher.
2073  * @param[in, out] key
2074  *   Flow matcher value.
2075  * @param[in] item
2076  *   Flow pattern to translate.
2077  * @param[in] inner
2078  *   Item is inner pattern.
2079  */
2080 static void
2081 flow_dv_translate_item_eth(void *matcher, void *key,
2082                            const struct rte_flow_item *item, int inner)
2083 {
2084         const struct rte_flow_item_eth *eth_m = item->mask;
2085         const struct rte_flow_item_eth *eth_v = item->spec;
2086         const struct rte_flow_item_eth nic_mask = {
2087                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2088                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2089                 .type = RTE_BE16(0xffff),
2090         };
2091         void *headers_m;
2092         void *headers_v;
2093         char *l24_v;
2094         unsigned int i;
2095
2096         if (!eth_v)
2097                 return;
2098         if (!eth_m)
2099                 eth_m = &nic_mask;
2100         if (inner) {
2101                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2102                                          inner_headers);
2103                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2104         } else {
2105                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2106                                          outer_headers);
2107                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2108         }
2109         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
2110                &eth_m->dst, sizeof(eth_m->dst));
2111         /* The value must be in the range of the mask. */
2112         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
2113         for (i = 0; i < sizeof(eth_m->dst); ++i)
2114                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
2115         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
2116                &eth_m->src, sizeof(eth_m->src));
2117         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
2118         /* The value must be in the range of the mask. */
2119         for (i = 0; i < sizeof(eth_m->dst); ++i)
2120                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
2121         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
2122                  rte_be_to_cpu_16(eth_m->type));
2123         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
2124         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
2125 }
2126
2127 /**
2128  * Add VLAN item to matcher and to the value.
2129  *
2130  * @param[in, out] matcher
2131  *   Flow matcher.
2132  * @param[in, out] key
2133  *   Flow matcher value.
2134  * @param[in] item
2135  *   Flow pattern to translate.
2136  * @param[in] inner
2137  *   Item is inner pattern.
2138  */
2139 static void
2140 flow_dv_translate_item_vlan(void *matcher, void *key,
2141                             const struct rte_flow_item *item,
2142                             int inner)
2143 {
2144         const struct rte_flow_item_vlan *vlan_m = item->mask;
2145         const struct rte_flow_item_vlan *vlan_v = item->spec;
2146         const struct rte_flow_item_vlan nic_mask = {
2147                 .tci = RTE_BE16(0x0fff),
2148                 .inner_type = RTE_BE16(0xffff),
2149         };
2150         void *headers_m;
2151         void *headers_v;
2152         uint16_t tci_m;
2153         uint16_t tci_v;
2154
2155         if (!vlan_v)
2156                 return;
2157         if (!vlan_m)
2158                 vlan_m = &nic_mask;
2159         if (inner) {
2160                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2161                                          inner_headers);
2162                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2163         } else {
2164                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2165                                          outer_headers);
2166                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2167         }
2168         tci_m = rte_be_to_cpu_16(vlan_m->tci);
2169         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
2170         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
2171         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
2172         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
2173         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
2174         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
2175         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
2176         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
2177         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
2178 }
2179
2180 /**
2181  * Add IPV4 item to matcher and to the value.
2182  *
2183  * @param[in, out] matcher
2184  *   Flow matcher.
2185  * @param[in, out] key
2186  *   Flow matcher value.
2187  * @param[in] item
2188  *   Flow pattern to translate.
2189  * @param[in] inner
2190  *   Item is inner pattern.
2191  * @param[in] group
2192  *   The group to insert the rule.
2193  */
2194 static void
2195 flow_dv_translate_item_ipv4(void *matcher, void *key,
2196                             const struct rte_flow_item *item,
2197                             int inner, uint32_t group)
2198 {
2199         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
2200         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
2201         const struct rte_flow_item_ipv4 nic_mask = {
2202                 .hdr = {
2203                         .src_addr = RTE_BE32(0xffffffff),
2204                         .dst_addr = RTE_BE32(0xffffffff),
2205                         .type_of_service = 0xff,
2206                         .next_proto_id = 0xff,
2207                 },
2208         };
2209         void *headers_m;
2210         void *headers_v;
2211         char *l24_m;
2212         char *l24_v;
2213         uint8_t tos;
2214
2215         if (inner) {
2216                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2217                                          inner_headers);
2218                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2219         } else {
2220                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2221                                          outer_headers);
2222                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2223         }
2224         if (group == 0)
2225                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
2226         else
2227                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
2228         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
2229         if (!ipv4_v)
2230                 return;
2231         if (!ipv4_m)
2232                 ipv4_m = &nic_mask;
2233         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2234                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2235         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2236                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2237         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
2238         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
2239         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2240                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
2241         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2242                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
2243         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
2244         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
2245         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
2246         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
2247                  ipv4_m->hdr.type_of_service);
2248         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
2249         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
2250                  ipv4_m->hdr.type_of_service >> 2);
2251         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
2252         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
2253                  ipv4_m->hdr.next_proto_id);
2254         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2255                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
2256 }
2257
2258 /**
2259  * Add IPV6 item to matcher and to the value.
2260  *
2261  * @param[in, out] matcher
2262  *   Flow matcher.
2263  * @param[in, out] key
2264  *   Flow matcher value.
2265  * @param[in] item
2266  *   Flow pattern to translate.
2267  * @param[in] inner
2268  *   Item is inner pattern.
2269  * @param[in] group
2270  *   The group to insert the rule.
2271  */
2272 static void
2273 flow_dv_translate_item_ipv6(void *matcher, void *key,
2274                             const struct rte_flow_item *item,
2275                             int inner, uint32_t group)
2276 {
2277         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
2278         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
2279         const struct rte_flow_item_ipv6 nic_mask = {
2280                 .hdr = {
2281                         .src_addr =
2282                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2283                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2284                         .dst_addr =
2285                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2286                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2287                         .vtc_flow = RTE_BE32(0xffffffff),
2288                         .proto = 0xff,
2289                         .hop_limits = 0xff,
2290                 },
2291         };
2292         void *headers_m;
2293         void *headers_v;
2294         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2295         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2296         char *l24_m;
2297         char *l24_v;
2298         uint32_t vtc_m;
2299         uint32_t vtc_v;
2300         int i;
2301         int size;
2302
2303         if (inner) {
2304                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2305                                          inner_headers);
2306                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2307         } else {
2308                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2309                                          outer_headers);
2310                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2311         }
2312         if (group == 0)
2313                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
2314         else
2315                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
2316         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
2317         if (!ipv6_v)
2318                 return;
2319         if (!ipv6_m)
2320                 ipv6_m = &nic_mask;
2321         size = sizeof(ipv6_m->hdr.dst_addr);
2322         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2323                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
2324         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2325                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
2326         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
2327         for (i = 0; i < size; ++i)
2328                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
2329         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2330                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
2331         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2332                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
2333         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
2334         for (i = 0; i < size; ++i)
2335                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
2336         /* TOS. */
2337         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
2338         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
2339         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
2340         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
2341         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
2342         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
2343         /* Label. */
2344         if (inner) {
2345                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
2346                          vtc_m);
2347                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
2348                          vtc_v);
2349         } else {
2350                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
2351                          vtc_m);
2352                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
2353                          vtc_v);
2354         }
2355         /* Protocol. */
2356         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
2357                  ipv6_m->hdr.proto);
2358         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2359                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
2360 }
2361
2362 /**
2363  * Add TCP item to matcher and to the value.
2364  *
2365  * @param[in, out] matcher
2366  *   Flow matcher.
2367  * @param[in, out] key
2368  *   Flow matcher value.
2369  * @param[in] item
2370  *   Flow pattern to translate.
2371  * @param[in] inner
2372  *   Item is inner pattern.
2373  */
2374 static void
2375 flow_dv_translate_item_tcp(void *matcher, void *key,
2376                            const struct rte_flow_item *item,
2377                            int inner)
2378 {
2379         const struct rte_flow_item_tcp *tcp_m = item->mask;
2380         const struct rte_flow_item_tcp *tcp_v = item->spec;
2381         void *headers_m;
2382         void *headers_v;
2383
2384         if (inner) {
2385                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2386                                          inner_headers);
2387                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2388         } else {
2389                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2390                                          outer_headers);
2391                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2392         }
2393         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2394         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
2395         if (!tcp_v)
2396                 return;
2397         if (!tcp_m)
2398                 tcp_m = &rte_flow_item_tcp_mask;
2399         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
2400                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
2401         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
2402                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
2403         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
2404                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
2405         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
2406                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
2407 }
2408
2409 /**
2410  * Add UDP item to matcher and to the value.
2411  *
2412  * @param[in, out] matcher
2413  *   Flow matcher.
2414  * @param[in, out] key
2415  *   Flow matcher value.
2416  * @param[in] item
2417  *   Flow pattern to translate.
2418  * @param[in] inner
2419  *   Item is inner pattern.
2420  */
2421 static void
2422 flow_dv_translate_item_udp(void *matcher, void *key,
2423                            const struct rte_flow_item *item,
2424                            int inner)
2425 {
2426         const struct rte_flow_item_udp *udp_m = item->mask;
2427         const struct rte_flow_item_udp *udp_v = item->spec;
2428         void *headers_m;
2429         void *headers_v;
2430
2431         if (inner) {
2432                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2433                                          inner_headers);
2434                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2435         } else {
2436                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2437                                          outer_headers);
2438                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2439         }
2440         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2441         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
2442         if (!udp_v)
2443                 return;
2444         if (!udp_m)
2445                 udp_m = &rte_flow_item_udp_mask;
2446         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
2447                  rte_be_to_cpu_16(udp_m->hdr.src_port));
2448         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
2449                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
2450         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
2451                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
2452         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2453                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
2454 }
2455
2456 /**
2457  * Add GRE item to matcher and to the value.
2458  *
2459  * @param[in, out] matcher
2460  *   Flow matcher.
2461  * @param[in, out] key
2462  *   Flow matcher value.
2463  * @param[in] item
2464  *   Flow pattern to translate.
2465  * @param[in] inner
2466  *   Item is inner pattern.
2467  */
2468 static void
2469 flow_dv_translate_item_gre(void *matcher, void *key,
2470                            const struct rte_flow_item *item,
2471                            int inner)
2472 {
2473         const struct rte_flow_item_gre *gre_m = item->mask;
2474         const struct rte_flow_item_gre *gre_v = item->spec;
2475         void *headers_m;
2476         void *headers_v;
2477         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2478         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2479
2480         if (inner) {
2481                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2482                                          inner_headers);
2483                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2484         } else {
2485                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2486                                          outer_headers);
2487                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2488         }
2489         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2490         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
2491         if (!gre_v)
2492                 return;
2493         if (!gre_m)
2494                 gre_m = &rte_flow_item_gre_mask;
2495         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
2496                  rte_be_to_cpu_16(gre_m->protocol));
2497         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
2498                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
2499 }
2500
2501 /**
2502  * Add NVGRE item to matcher and to the value.
2503  *
2504  * @param[in, out] matcher
2505  *   Flow matcher.
2506  * @param[in, out] key
2507  *   Flow matcher value.
2508  * @param[in] item
2509  *   Flow pattern to translate.
2510  * @param[in] inner
2511  *   Item is inner pattern.
2512  */
2513 static void
2514 flow_dv_translate_item_nvgre(void *matcher, void *key,
2515                              const struct rte_flow_item *item,
2516                              int inner)
2517 {
2518         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
2519         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
2520         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2521         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2522         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
2523         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
2524         char *gre_key_m;
2525         char *gre_key_v;
2526         int size;
2527         int i;
2528
2529         flow_dv_translate_item_gre(matcher, key, item, inner);
2530         if (!nvgre_v)
2531                 return;
2532         if (!nvgre_m)
2533                 nvgre_m = &rte_flow_item_nvgre_mask;
2534         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
2535         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
2536         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
2537         memcpy(gre_key_m, tni_flow_id_m, size);
2538         for (i = 0; i < size; ++i)
2539                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
2540 }
2541
2542 /**
2543  * Add VXLAN item to matcher and to the value.
2544  *
2545  * @param[in, out] matcher
2546  *   Flow matcher.
2547  * @param[in, out] key
2548  *   Flow matcher value.
2549  * @param[in] item
2550  *   Flow pattern to translate.
2551  * @param[in] inner
2552  *   Item is inner pattern.
2553  */
2554 static void
2555 flow_dv_translate_item_vxlan(void *matcher, void *key,
2556                              const struct rte_flow_item *item,
2557                              int inner)
2558 {
2559         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
2560         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
2561         void *headers_m;
2562         void *headers_v;
2563         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2564         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2565         char *vni_m;
2566         char *vni_v;
2567         uint16_t dport;
2568         int size;
2569         int i;
2570
2571         if (inner) {
2572                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2573                                          inner_headers);
2574                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2575         } else {
2576                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2577                                          outer_headers);
2578                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2579         }
2580         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
2581                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
2582         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
2583                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
2584                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
2585         }
2586         if (!vxlan_v)
2587                 return;
2588         if (!vxlan_m)
2589                 vxlan_m = &rte_flow_item_vxlan_mask;
2590         size = sizeof(vxlan_m->vni);
2591         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
2592         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
2593         memcpy(vni_m, vxlan_m->vni, size);
2594         for (i = 0; i < size; ++i)
2595                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
2596 }
2597
2598 /**
2599  * Add MPLS item to matcher and to the value.
2600  *
2601  * @param[in, out] matcher
2602  *   Flow matcher.
2603  * @param[in, out] key
2604  *   Flow matcher value.
2605  * @param[in] item
2606  *   Flow pattern to translate.
2607  * @param[in] prev_layer
2608  *   The protocol layer indicated in previous item.
2609  * @param[in] inner
2610  *   Item is inner pattern.
2611  */
2612 static void
2613 flow_dv_translate_item_mpls(void *matcher, void *key,
2614                             const struct rte_flow_item *item,
2615                             uint64_t prev_layer,
2616                             int inner)
2617 {
2618         const uint32_t *in_mpls_m = item->mask;
2619         const uint32_t *in_mpls_v = item->spec;
2620         uint32_t *out_mpls_m = 0;
2621         uint32_t *out_mpls_v = 0;
2622         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2623         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2624         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
2625                                      misc_parameters_2);
2626         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
2627         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
2628         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2629
2630         switch (prev_layer) {
2631         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
2632                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
2633                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2634                          MLX5_UDP_PORT_MPLS);
2635                 break;
2636         case MLX5_FLOW_LAYER_GRE:
2637                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
2638                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
2639                          ETHER_TYPE_MPLS);
2640                 break;
2641         default:
2642                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2643                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2644                          IPPROTO_MPLS);
2645                 break;
2646         }
2647         if (!in_mpls_v)
2648                 return;
2649         if (!in_mpls_m)
2650                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
2651         switch (prev_layer) {
2652         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
2653                 out_mpls_m =
2654                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
2655                                                  outer_first_mpls_over_udp);
2656                 out_mpls_v =
2657                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
2658                                                  outer_first_mpls_over_udp);
2659                 break;
2660         case MLX5_FLOW_LAYER_GRE:
2661                 out_mpls_m =
2662                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
2663                                                  outer_first_mpls_over_gre);
2664                 out_mpls_v =
2665                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
2666                                                  outer_first_mpls_over_gre);
2667                 break;
2668         default:
2669                 /* Inner MPLS not over GRE is not supported. */
2670                 if (!inner) {
2671                         out_mpls_m =
2672                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
2673                                                          misc2_m,
2674                                                          outer_first_mpls);
2675                         out_mpls_v =
2676                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
2677                                                          misc2_v,
2678                                                          outer_first_mpls);
2679                 }
2680                 break;
2681         }
2682         if (out_mpls_m && out_mpls_v) {
2683                 *out_mpls_m = *in_mpls_m;
2684                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
2685         }
2686 }
2687
2688 /**
2689  * Add META item to matcher
2690  *
2691  * @param[in, out] matcher
2692  *   Flow matcher.
2693  * @param[in, out] key
2694  *   Flow matcher value.
2695  * @param[in] item
2696  *   Flow pattern to translate.
2697  * @param[in] inner
2698  *   Item is inner pattern.
2699  */
2700 static void
2701 flow_dv_translate_item_meta(void *matcher, void *key,
2702                             const struct rte_flow_item *item)
2703 {
2704         const struct rte_flow_item_meta *meta_m;
2705         const struct rte_flow_item_meta *meta_v;
2706         void *misc2_m =
2707                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
2708         void *misc2_v =
2709                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
2710
2711         meta_m = (const void *)item->mask;
2712         if (!meta_m)
2713                 meta_m = &rte_flow_item_meta_mask;
2714         meta_v = (const void *)item->spec;
2715         if (meta_v) {
2716                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
2717                          rte_be_to_cpu_32(meta_m->data));
2718                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
2719                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
2720         }
2721 }
2722
2723 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
2724
2725 #define HEADER_IS_ZERO(match_criteria, headers)                              \
2726         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
2727                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
2728
2729 /**
2730  * Calculate flow matcher enable bitmap.
2731  *
2732  * @param match_criteria
2733  *   Pointer to flow matcher criteria.
2734  *
2735  * @return
2736  *   Bitmap of enabled fields.
2737  */
2738 static uint8_t
2739 flow_dv_matcher_enable(uint32_t *match_criteria)
2740 {
2741         uint8_t match_criteria_enable;
2742
2743         match_criteria_enable =
2744                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
2745                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
2746         match_criteria_enable |=
2747                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
2748                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
2749         match_criteria_enable |=
2750                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
2751                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
2752         match_criteria_enable |=
2753                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
2754                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
2755 #ifdef HAVE_MLX5DV_DR
2756         match_criteria_enable |=
2757                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
2758                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
2759 #endif
2760         return match_criteria_enable;
2761 }
2762
2763 /**
2764  * Register the flow matcher.
2765  *
2766  * @param dev[in, out]
2767  *   Pointer to rte_eth_dev structure.
2768  * @param[in, out] matcher
2769  *   Pointer to flow matcher.
2770  * @parm[in, out] dev_flow
2771  *   Pointer to the dev_flow.
2772  * @param[out] error
2773  *   pointer to error structure.
2774  *
2775  * @return
2776  *   0 on success otherwise -errno and errno is set.
2777  */
2778 static int
2779 flow_dv_matcher_register(struct rte_eth_dev *dev,
2780                          struct mlx5_flow_dv_matcher *matcher,
2781                          struct mlx5_flow *dev_flow,
2782                          struct rte_flow_error *error)
2783 {
2784         struct mlx5_priv *priv = dev->data->dev_private;
2785         struct mlx5_flow_dv_matcher *cache_matcher;
2786         struct mlx5dv_flow_matcher_attr dv_attr = {
2787                 .type = IBV_FLOW_ATTR_NORMAL,
2788                 .match_mask = (void *)&matcher->mask,
2789         };
2790         struct mlx5_flow_tbl_resource *tbl = NULL;
2791
2792         /* Lookup from cache. */
2793         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
2794                 if (matcher->crc == cache_matcher->crc &&
2795                     matcher->priority == cache_matcher->priority &&
2796                     matcher->egress == cache_matcher->egress &&
2797                     matcher->group == cache_matcher->group &&
2798                     !memcmp((const void *)matcher->mask.buf,
2799                             (const void *)cache_matcher->mask.buf,
2800                             cache_matcher->mask.size)) {
2801                         DRV_LOG(DEBUG,
2802                                 "priority %hd use %s matcher %p: refcnt %d++",
2803                                 cache_matcher->priority,
2804                                 cache_matcher->egress ? "tx" : "rx",
2805                                 (void *)cache_matcher,
2806                                 rte_atomic32_read(&cache_matcher->refcnt));
2807                         rte_atomic32_inc(&cache_matcher->refcnt);
2808                         dev_flow->dv.matcher = cache_matcher;
2809                         return 0;
2810                 }
2811         }
2812 #ifdef HAVE_MLX5DV_DR
2813         if (matcher->egress) {
2814                 tbl = &priv->tx_tbl[matcher->group];
2815                 if (!tbl->obj)
2816                         tbl->obj = mlx5_glue->dr_create_flow_tbl
2817                                 (priv->tx_ns,
2818                                  matcher->group * MLX5_GROUP_FACTOR);
2819         } else {
2820                 tbl = &priv->rx_tbl[matcher->group];
2821                 if (!tbl->obj)
2822                         tbl->obj = mlx5_glue->dr_create_flow_tbl
2823                                 (priv->rx_ns,
2824                                  matcher->group * MLX5_GROUP_FACTOR);
2825         }
2826         if (!tbl->obj)
2827                 return rte_flow_error_set(error, ENOMEM,
2828                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2829                                           NULL, "cannot create table");
2830
2831         rte_atomic32_inc(&tbl->refcnt);
2832 #endif
2833         /* Register new matcher. */
2834         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
2835         if (!cache_matcher)
2836                 return rte_flow_error_set(error, ENOMEM,
2837                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2838                                           "cannot allocate matcher memory");
2839         *cache_matcher = *matcher;
2840         dv_attr.match_criteria_enable =
2841                 flow_dv_matcher_enable(cache_matcher->mask.buf);
2842         dv_attr.priority = matcher->priority;
2843         if (matcher->egress)
2844                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
2845         cache_matcher->matcher_object =
2846                 mlx5_glue->dv_create_flow_matcher(priv->sh->ctx, &dv_attr,
2847                                                   tbl->obj);
2848         if (!cache_matcher->matcher_object) {
2849                 rte_free(cache_matcher);
2850 #ifdef HAVE_MLX5DV_DR
2851                 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
2852                         mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
2853                         tbl->obj = NULL;
2854                 }
2855 #endif
2856                 return rte_flow_error_set(error, ENOMEM,
2857                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2858                                           NULL, "cannot create matcher");
2859         }
2860         rte_atomic32_inc(&cache_matcher->refcnt);
2861         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
2862         dev_flow->dv.matcher = cache_matcher;
2863         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
2864                 cache_matcher->priority,
2865                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
2866                 rte_atomic32_read(&cache_matcher->refcnt));
2867         rte_atomic32_inc(&tbl->refcnt);
2868         return 0;
2869 }
2870
2871 /**
2872  * Add source vport match to the specified matcher.
2873  *
2874  * @param[in, out] matcher
2875  *   Flow matcher.
2876  * @param[in, out] key
2877  *   Flow matcher value.
2878  * @param[in] port
2879  *   Source vport value to match
2880  * @param[in] mask
2881  *   Mask
2882  */
2883 static void
2884 flow_dv_translate_source_vport(void *matcher, void *key,
2885                               int16_t port, uint16_t mask)
2886 {
2887         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2888         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2889
2890         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
2891         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
2892 }
2893
2894 /**
2895  * Find existing tag resource or create and register a new one.
2896  *
2897  * @param dev[in, out]
2898  *   Pointer to rte_eth_dev structure.
2899  * @param[in, out] resource
2900  *   Pointer to tag resource.
2901  * @parm[in, out] dev_flow
2902  *   Pointer to the dev_flow.
2903  * @param[out] error
2904  *   pointer to error structure.
2905  *
2906  * @return
2907  *   0 on success otherwise -errno and errno is set.
2908  */
2909 static int
2910 flow_dv_tag_resource_register
2911                         (struct rte_eth_dev *dev,
2912                          struct mlx5_flow_dv_tag_resource *resource,
2913                          struct mlx5_flow *dev_flow,
2914                          struct rte_flow_error *error)
2915 {
2916         struct mlx5_priv *priv = dev->data->dev_private;
2917         struct mlx5_flow_dv_tag_resource *cache_resource;
2918
2919         /* Lookup a matching resource from cache. */
2920         LIST_FOREACH(cache_resource, &priv->tags, next) {
2921                 if (resource->tag == cache_resource->tag) {
2922                         DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
2923                                 (void *)cache_resource,
2924                                 rte_atomic32_read(&cache_resource->refcnt));
2925                         rte_atomic32_inc(&cache_resource->refcnt);
2926                         dev_flow->flow->tag_resource = cache_resource;
2927                         return 0;
2928                 }
2929         }
2930         /* Register new  resource. */
2931         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2932         if (!cache_resource)
2933                 return rte_flow_error_set(error, ENOMEM,
2934                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2935                                           "cannot allocate resource memory");
2936         *cache_resource = *resource;
2937         cache_resource->action = mlx5_glue->dv_create_flow_action_tag
2938                 (resource->tag);
2939         if (!cache_resource->action) {
2940                 rte_free(cache_resource);
2941                 return rte_flow_error_set(error, ENOMEM,
2942                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2943                                           NULL, "cannot create action");
2944         }
2945         rte_atomic32_init(&cache_resource->refcnt);
2946         rte_atomic32_inc(&cache_resource->refcnt);
2947         LIST_INSERT_HEAD(&priv->tags, cache_resource, next);
2948         dev_flow->flow->tag_resource = cache_resource;
2949         DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
2950                 (void *)cache_resource,
2951                 rte_atomic32_read(&cache_resource->refcnt));
2952         return 0;
2953 }
2954
2955 /**
2956  * Release the tag.
2957  *
2958  * @param dev
2959  *   Pointer to Ethernet device.
2960  * @param flow
2961  *   Pointer to mlx5_flow.
2962  *
2963  * @return
2964  *   1 while a reference on it exists, 0 when freed.
2965  */
2966 static int
2967 flow_dv_tag_release(struct rte_eth_dev *dev,
2968                     struct mlx5_flow_dv_tag_resource *tag)
2969 {
2970         assert(tag);
2971         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
2972                 dev->data->port_id, (void *)tag,
2973                 rte_atomic32_read(&tag->refcnt));
2974         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
2975                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
2976                 LIST_REMOVE(tag, next);
2977                 DRV_LOG(DEBUG, "port %u tag %p: removed",
2978                         dev->data->port_id, (void *)tag);
2979                 rte_free(tag);
2980                 return 0;
2981         }
2982         return 1;
2983 }
2984
2985 /**
2986  * Fill the flow with DV spec.
2987  *
2988  * @param[in] dev
2989  *   Pointer to rte_eth_dev structure.
2990  * @param[in, out] dev_flow
2991  *   Pointer to the sub flow.
2992  * @param[in] attr
2993  *   Pointer to the flow attributes.
2994  * @param[in] items
2995  *   Pointer to the list of items.
2996  * @param[in] actions
2997  *   Pointer to the list of actions.
2998  * @param[out] error
2999  *   Pointer to the error structure.
3000  *
3001  * @return
3002  *   0 on success, a negative errno value otherwise and rte_errno is set.
3003  */
3004 static int
3005 flow_dv_translate(struct rte_eth_dev *dev,
3006                   struct mlx5_flow *dev_flow,
3007                   const struct rte_flow_attr *attr,
3008                   const struct rte_flow_item items[],
3009                   const struct rte_flow_action actions[],
3010                   struct rte_flow_error *error)
3011 {
3012         struct mlx5_priv *priv = dev->data->dev_private;
3013         struct rte_flow *flow = dev_flow->flow;
3014         uint64_t item_flags = 0;
3015         uint64_t last_item = 0;
3016         uint64_t action_flags = 0;
3017         uint64_t priority = attr->priority;
3018         struct mlx5_flow_dv_matcher matcher = {
3019                 .mask = {
3020                         .size = sizeof(matcher.mask.buf),
3021                 },
3022         };
3023         int actions_n = 0;
3024         bool actions_end = false;
3025         struct mlx5_flow_dv_modify_hdr_resource res = {
3026                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3027                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
3028         };
3029         union flow_dv_attr flow_attr = { .attr = 0 };
3030         struct mlx5_flow_dv_tag_resource tag_resource;
3031
3032         if (priority == MLX5_FLOW_PRIO_RSVD)
3033                 priority = priv->config.flow_prio - 1;
3034         for (; !actions_end ; actions++) {
3035                 const struct rte_flow_action_queue *queue;
3036                 const struct rte_flow_action_rss *rss;
3037                 const struct rte_flow_action *action = actions;
3038                 const struct rte_flow_action_count *count = action->conf;
3039                 const uint8_t *rss_key;
3040
3041                 switch (actions->type) {
3042                 case RTE_FLOW_ACTION_TYPE_VOID:
3043                         break;
3044                 case RTE_FLOW_ACTION_TYPE_FLAG:
3045                         tag_resource.tag =
3046                                 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
3047                         if (!flow->tag_resource)
3048                                 if (flow_dv_tag_resource_register
3049                                     (dev, &tag_resource, dev_flow, error))
3050                                         return errno;
3051                         dev_flow->dv.actions[actions_n++] =
3052                                 flow->tag_resource->action;
3053                         action_flags |= MLX5_FLOW_ACTION_FLAG;
3054                         break;
3055                 case RTE_FLOW_ACTION_TYPE_MARK:
3056                         tag_resource.tag = mlx5_flow_mark_set
3057                               (((const struct rte_flow_action_mark *)
3058                                (actions->conf))->id);
3059                         if (!flow->tag_resource)
3060                                 if (flow_dv_tag_resource_register
3061                                     (dev, &tag_resource, dev_flow, error))
3062                                         return errno;
3063                         dev_flow->dv.actions[actions_n++] =
3064                                 flow->tag_resource->action;
3065                         action_flags |= MLX5_FLOW_ACTION_MARK;
3066                         break;
3067                 case RTE_FLOW_ACTION_TYPE_DROP:
3068                         action_flags |= MLX5_FLOW_ACTION_DROP;
3069                         break;
3070                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3071                         queue = actions->conf;
3072                         flow->rss.queue_num = 1;
3073                         (*flow->queue)[0] = queue->index;
3074                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
3075                         break;
3076                 case RTE_FLOW_ACTION_TYPE_RSS:
3077                         rss = actions->conf;
3078                         if (flow->queue)
3079                                 memcpy((*flow->queue), rss->queue,
3080                                        rss->queue_num * sizeof(uint16_t));
3081                         flow->rss.queue_num = rss->queue_num;
3082                         /* NULL RSS key indicates default RSS key. */
3083                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
3084                         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
3085                         /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
3086                         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
3087                         flow->rss.level = rss->level;
3088                         action_flags |= MLX5_FLOW_ACTION_RSS;
3089                         break;
3090                 case RTE_FLOW_ACTION_TYPE_COUNT:
3091                         if (!priv->config.devx) {
3092                                 rte_errno = ENOTSUP;
3093                                 goto cnt_err;
3094                         }
3095                         flow->counter = flow_dv_counter_new(dev, count->shared,
3096                                                             count->id);
3097                         if (flow->counter == NULL)
3098                                 goto cnt_err;
3099                         dev_flow->dv.actions[actions_n++] =
3100                                 flow->counter->action;
3101                         action_flags |= MLX5_FLOW_ACTION_COUNT;
3102                         break;
3103 cnt_err:
3104                         if (rte_errno == ENOTSUP)
3105                                 return rte_flow_error_set
3106                                               (error, ENOTSUP,
3107                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3108                                                NULL,
3109                                                "count action not supported");
3110                         else
3111                                 return rte_flow_error_set
3112                                                 (error, rte_errno,
3113                                                  RTE_FLOW_ERROR_TYPE_ACTION,
3114                                                  action,
3115                                                  "cannot create counter"
3116                                                   " object.");
3117                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3118                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3119                         if (flow_dv_create_action_l2_encap(dev, actions,
3120                                                            dev_flow, error))
3121                                 return -rte_errno;
3122                         dev_flow->dv.actions[actions_n++] =
3123                                 dev_flow->dv.encap_decap->verbs_action;
3124                         action_flags |= actions->type ==
3125                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3126                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
3127                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
3128                         break;
3129                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3130                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3131                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
3132                                                            error))
3133                                 return -rte_errno;
3134                         dev_flow->dv.actions[actions_n++] =
3135                                 dev_flow->dv.encap_decap->verbs_action;
3136                         action_flags |= actions->type ==
3137                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3138                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
3139                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
3140                         break;
3141                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3142                         /* Handle encap with preceding decap. */
3143                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
3144                                 if (flow_dv_create_action_raw_encap
3145                                         (dev, actions, dev_flow, attr, error))
3146                                         return -rte_errno;
3147                                 dev_flow->dv.actions[actions_n++] =
3148                                         dev_flow->dv.encap_decap->verbs_action;
3149                         } else {
3150                                 /* Handle encap without preceding decap. */
3151                                 if (flow_dv_create_action_l2_encap(dev, actions,
3152                                                                    dev_flow,
3153                                                                    error))
3154                                         return -rte_errno;
3155                                 dev_flow->dv.actions[actions_n++] =
3156                                         dev_flow->dv.encap_decap->verbs_action;
3157                         }
3158                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3159                         break;
3160                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3161                         /* Check if this decap is followed by encap. */
3162                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
3163                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
3164                                action++) {
3165                         }
3166                         /* Handle decap only if it isn't followed by encap. */
3167                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3168                                 if (flow_dv_create_action_l2_decap(dev,
3169                                                                    dev_flow,
3170                                                                    error))
3171                                         return -rte_errno;
3172                                 dev_flow->dv.actions[actions_n++] =
3173                                         dev_flow->dv.encap_decap->verbs_action;
3174                         }
3175                         /* If decap is followed by encap, handle it at encap. */
3176                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3177                         break;
3178                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3179                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3180                         if (flow_dv_convert_action_modify_mac(&res, actions,
3181                                                               error))
3182                                 return -rte_errno;
3183                         action_flags |= actions->type ==
3184                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3185                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
3186                                         MLX5_FLOW_ACTION_SET_MAC_DST;
3187                         break;
3188                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3189                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3190                         if (flow_dv_convert_action_modify_ipv4(&res, actions,
3191                                                                error))
3192                                 return -rte_errno;
3193                         action_flags |= actions->type ==
3194                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3195                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
3196                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
3197                         break;
3198                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3199                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3200                         if (flow_dv_convert_action_modify_ipv6(&res, actions,
3201                                                                error))
3202                                 return -rte_errno;
3203                         action_flags |= actions->type ==
3204                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3205                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
3206                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
3207                         break;
3208                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3209                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3210                         if (flow_dv_convert_action_modify_tp(&res, actions,
3211                                                              items, &flow_attr,
3212                                                              error))
3213                                 return -rte_errno;
3214                         action_flags |= actions->type ==
3215                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3216                                         MLX5_FLOW_ACTION_SET_TP_SRC :
3217                                         MLX5_FLOW_ACTION_SET_TP_DST;
3218                         break;
3219                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3220                         if (flow_dv_convert_action_modify_dec_ttl(&res, items,
3221                                                                   &flow_attr,
3222                                                                   error))
3223                                 return -rte_errno;
3224                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
3225                         break;
3226                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3227                         if (flow_dv_convert_action_modify_ttl(&res, actions,
3228                                                              items, &flow_attr,
3229                                                              error))
3230                                 return -rte_errno;
3231                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
3232                         break;
3233                 case RTE_FLOW_ACTION_TYPE_END:
3234                         actions_end = true;
3235                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
3236                                 /* create modify action if needed. */
3237                                 if (flow_dv_modify_hdr_resource_register
3238                                                                 (dev, &res,
3239                                                                  dev_flow,
3240                                                                  error))
3241                                         return -rte_errno;
3242                                 dev_flow->dv.actions[actions_n++] =
3243                                         dev_flow->dv.modify_hdr->verbs_action;
3244                         }
3245                         break;
3246                 default:
3247                         break;
3248                 }
3249         }
3250         dev_flow->dv.actions_n = actions_n;
3251         flow->actions = action_flags;
3252         if (attr->ingress && !attr->transfer &&
3253             (priv->representor || priv->master)) {
3254                 /* It was validated - we support unidirection flows only. */
3255                 assert(!attr->egress);
3256                 /*
3257                  * Add matching on source vport index only
3258                  * for ingress rules in E-Switch configurations.
3259                  */
3260                 flow_dv_translate_source_vport(matcher.mask.buf,
3261                                                dev_flow->dv.value.buf,
3262                                                priv->vport_id,
3263                                                0xffff);
3264         }
3265         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3266                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3267                 void *match_mask = matcher.mask.buf;
3268                 void *match_value = dev_flow->dv.value.buf;
3269
3270                 switch (items->type) {
3271                 case RTE_FLOW_ITEM_TYPE_ETH:
3272                         flow_dv_translate_item_eth(match_mask, match_value,
3273                                                    items, tunnel);
3274                         matcher.priority = MLX5_PRIORITY_MAP_L2;
3275                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
3276                                              MLX5_FLOW_LAYER_OUTER_L2;
3277                         break;
3278                 case RTE_FLOW_ITEM_TYPE_VLAN:
3279                         flow_dv_translate_item_vlan(match_mask, match_value,
3280                                                     items, tunnel);
3281                         matcher.priority = MLX5_PRIORITY_MAP_L2;
3282                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
3283                                               MLX5_FLOW_LAYER_INNER_VLAN) :
3284                                              (MLX5_FLOW_LAYER_OUTER_L2 |
3285                                               MLX5_FLOW_LAYER_OUTER_VLAN);
3286                         break;
3287                 case RTE_FLOW_ITEM_TYPE_IPV4:
3288                         flow_dv_translate_item_ipv4(match_mask, match_value,
3289                                                     items, tunnel, attr->group);
3290                         matcher.priority = MLX5_PRIORITY_MAP_L3;
3291                         dev_flow->dv.hash_fields |=
3292                                 mlx5_flow_hashfields_adjust
3293                                         (dev_flow, tunnel,
3294                                          MLX5_IPV4_LAYER_TYPES,
3295                                          MLX5_IPV4_IBV_RX_HASH);
3296                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3297                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3298                         break;
3299                 case RTE_FLOW_ITEM_TYPE_IPV6:
3300                         flow_dv_translate_item_ipv6(match_mask, match_value,
3301                                                     items, tunnel, attr->group);
3302                         matcher.priority = MLX5_PRIORITY_MAP_L3;
3303                         dev_flow->dv.hash_fields |=
3304                                 mlx5_flow_hashfields_adjust
3305                                         (dev_flow, tunnel,
3306                                          MLX5_IPV6_LAYER_TYPES,
3307                                          MLX5_IPV6_IBV_RX_HASH);
3308                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3309                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3310                         break;
3311                 case RTE_FLOW_ITEM_TYPE_TCP:
3312                         flow_dv_translate_item_tcp(match_mask, match_value,
3313                                                    items, tunnel);
3314                         matcher.priority = MLX5_PRIORITY_MAP_L4;
3315                         dev_flow->dv.hash_fields |=
3316                                 mlx5_flow_hashfields_adjust
3317                                         (dev_flow, tunnel, ETH_RSS_TCP,
3318                                          IBV_RX_HASH_SRC_PORT_TCP |
3319                                          IBV_RX_HASH_DST_PORT_TCP);
3320                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3321                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
3322                         break;
3323                 case RTE_FLOW_ITEM_TYPE_UDP:
3324                         flow_dv_translate_item_udp(match_mask, match_value,
3325                                                    items, tunnel);
3326                         matcher.priority = MLX5_PRIORITY_MAP_L4;
3327                         dev_flow->dv.hash_fields |=
3328                                 mlx5_flow_hashfields_adjust
3329                                         (dev_flow, tunnel, ETH_RSS_UDP,
3330                                          IBV_RX_HASH_SRC_PORT_UDP |
3331                                          IBV_RX_HASH_DST_PORT_UDP);
3332                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3333                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
3334                         break;
3335                 case RTE_FLOW_ITEM_TYPE_GRE:
3336                         flow_dv_translate_item_gre(match_mask, match_value,
3337                                                    items, tunnel);
3338                         last_item = MLX5_FLOW_LAYER_GRE;
3339                         break;
3340                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3341                         flow_dv_translate_item_nvgre(match_mask, match_value,
3342                                                      items, tunnel);
3343                         last_item = MLX5_FLOW_LAYER_GRE;
3344                         break;
3345                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3346                         flow_dv_translate_item_vxlan(match_mask, match_value,
3347                                                      items, tunnel);
3348                         last_item = MLX5_FLOW_LAYER_VXLAN;
3349                         break;
3350                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3351                         flow_dv_translate_item_vxlan(match_mask, match_value,
3352                                                      items, tunnel);
3353                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3354                         break;
3355                 case RTE_FLOW_ITEM_TYPE_MPLS:
3356                         flow_dv_translate_item_mpls(match_mask, match_value,
3357                                                     items, last_item, tunnel);
3358                         last_item = MLX5_FLOW_LAYER_MPLS;
3359                         break;
3360                 case RTE_FLOW_ITEM_TYPE_META:
3361                         flow_dv_translate_item_meta(match_mask, match_value,
3362                                                     items);
3363                         last_item = MLX5_FLOW_ITEM_METADATA;
3364                         break;
3365                 default:
3366                         break;
3367                 }
3368                 item_flags |= last_item;
3369         }
3370         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
3371                                          dev_flow->dv.value.buf));
3372         dev_flow->layers = item_flags;
3373         /* Register matcher. */
3374         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
3375                                     matcher.mask.size);
3376         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
3377                                                      matcher.priority);
3378         matcher.egress = attr->egress;
3379         matcher.group = attr->group;
3380         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
3381                 return -rte_errno;
3382         return 0;
3383 }
3384
3385 /**
3386  * Apply the flow to the NIC.
3387  *
3388  * @param[in] dev
3389  *   Pointer to the Ethernet device structure.
3390  * @param[in, out] flow
3391  *   Pointer to flow structure.
3392  * @param[out] error
3393  *   Pointer to error structure.
3394  *
3395  * @return
3396  *   0 on success, a negative errno value otherwise and rte_errno is set.
3397  */
3398 static int
3399 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3400               struct rte_flow_error *error)
3401 {
3402         struct mlx5_flow_dv *dv;
3403         struct mlx5_flow *dev_flow;
3404         int n;
3405         int err;
3406
3407         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
3408                 dv = &dev_flow->dv;
3409                 n = dv->actions_n;
3410                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
3411                         dv->hrxq = mlx5_hrxq_drop_new(dev);
3412                         if (!dv->hrxq) {
3413                                 rte_flow_error_set
3414                                         (error, errno,
3415                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3416                                          "cannot get drop hash queue");
3417                                 goto error;
3418                         }
3419                         dv->actions[n++] =
3420                                 mlx5_glue->dv_create_flow_action_dest_ibv_qp
3421                                 (dv->hrxq->qp);
3422                 } else if (flow->actions &
3423                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
3424                         struct mlx5_hrxq *hrxq;
3425
3426                         hrxq = mlx5_hrxq_get(dev, flow->key,
3427                                              MLX5_RSS_HASH_KEY_LEN,
3428                                              dv->hash_fields,
3429                                              (*flow->queue),
3430                                              flow->rss.queue_num);
3431                         if (!hrxq)
3432                                 hrxq = mlx5_hrxq_new
3433                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
3434                                          dv->hash_fields, (*flow->queue),
3435                                          flow->rss.queue_num,
3436                                          !!(dev_flow->layers &
3437                                             MLX5_FLOW_LAYER_TUNNEL));
3438                         if (!hrxq) {
3439                                 rte_flow_error_set
3440                                         (error, rte_errno,
3441                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3442                                          "cannot get hash queue");
3443                                 goto error;
3444                         }
3445                         dv->hrxq = hrxq;
3446                         dv->actions[n++] =
3447                                 mlx5_glue->dv_create_flow_action_dest_ibv_qp
3448                                 (dv->hrxq->qp);
3449                 }
3450                 dv->flow =
3451                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
3452                                                   (void *)&dv->value, n,
3453                                                   dv->actions);
3454                 if (!dv->flow) {
3455                         rte_flow_error_set(error, errno,
3456                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3457                                            NULL,
3458                                            "hardware refuses to create flow");
3459                         goto error;
3460                 }
3461         }
3462         return 0;
3463 error:
3464         err = rte_errno; /* Save rte_errno before cleanup. */
3465         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
3466                 struct mlx5_flow_dv *dv = &dev_flow->dv;
3467                 if (dv->hrxq) {
3468                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
3469                                 mlx5_hrxq_drop_release(dev);
3470                         else
3471                                 mlx5_hrxq_release(dev, dv->hrxq);
3472                         dv->hrxq = NULL;
3473                 }
3474         }
3475         rte_errno = err; /* Restore rte_errno. */
3476         return -rte_errno;
3477 }
3478
3479 /**
3480  * Release the flow matcher.
3481  *
3482  * @param dev
3483  *   Pointer to Ethernet device.
3484  * @param flow
3485  *   Pointer to mlx5_flow.
3486  *
3487  * @return
3488  *   1 while a reference on it exists, 0 when freed.
3489  */
3490 static int
3491 flow_dv_matcher_release(struct rte_eth_dev *dev,
3492                         struct mlx5_flow *flow)
3493 {
3494         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
3495         struct mlx5_priv *priv = dev->data->dev_private;
3496         struct mlx5_flow_tbl_resource *tbl;
3497
3498         assert(matcher->matcher_object);
3499         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
3500                 dev->data->port_id, (void *)matcher,
3501                 rte_atomic32_read(&matcher->refcnt));
3502         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
3503                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
3504                            (matcher->matcher_object));
3505                 LIST_REMOVE(matcher, next);
3506                 if (matcher->egress)
3507                         tbl = &priv->tx_tbl[matcher->group];
3508                 else
3509                         tbl = &priv->rx_tbl[matcher->group];
3510                 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
3511                         mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
3512                         tbl->obj = NULL;
3513                 }
3514                 rte_free(matcher);
3515                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
3516                         dev->data->port_id, (void *)matcher);
3517                 return 0;
3518         }
3519         return 1;
3520 }
3521
3522 /**
3523  * Release an encap/decap resource.
3524  *
3525  * @param flow
3526  *   Pointer to mlx5_flow.
3527  *
3528  * @return
3529  *   1 while a reference on it exists, 0 when freed.
3530  */
3531 static int
3532 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
3533 {
3534         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
3535                                                 flow->dv.encap_decap;
3536
3537         assert(cache_resource->verbs_action);
3538         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
3539                 (void *)cache_resource,
3540                 rte_atomic32_read(&cache_resource->refcnt));
3541         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
3542                 claim_zero(mlx5_glue->destroy_flow_action
3543                                 (cache_resource->verbs_action));
3544                 LIST_REMOVE(cache_resource, next);
3545                 rte_free(cache_resource);
3546                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
3547                         (void *)cache_resource);
3548                 return 0;
3549         }
3550         return 1;
3551 }
3552
3553 /**
3554  * Release a modify-header resource.
3555  *
3556  * @param flow
3557  *   Pointer to mlx5_flow.
3558  *
3559  * @return
3560  *   1 while a reference on it exists, 0 when freed.
3561  */
3562 static int
3563 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
3564 {
3565         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
3566                                                 flow->dv.modify_hdr;
3567
3568         assert(cache_resource->verbs_action);
3569         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
3570                 (void *)cache_resource,
3571                 rte_atomic32_read(&cache_resource->refcnt));
3572         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
3573                 claim_zero(mlx5_glue->destroy_flow_action
3574                                 (cache_resource->verbs_action));
3575                 LIST_REMOVE(cache_resource, next);
3576                 rte_free(cache_resource);
3577                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
3578                         (void *)cache_resource);
3579                 return 0;
3580         }
3581         return 1;
3582 }
3583
3584 /**
3585  * Remove the flow from the NIC but keeps it in memory.
3586  *
3587  * @param[in] dev
3588  *   Pointer to Ethernet device.
3589  * @param[in, out] flow
3590  *   Pointer to flow structure.
3591  */
3592 static void
3593 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
3594 {
3595         struct mlx5_flow_dv *dv;
3596         struct mlx5_flow *dev_flow;
3597
3598         if (!flow)
3599                 return;
3600         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
3601                 dv = &dev_flow->dv;
3602                 if (dv->flow) {
3603                         claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
3604                         dv->flow = NULL;
3605                 }
3606                 if (dv->hrxq) {
3607                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
3608                                 mlx5_hrxq_drop_release(dev);
3609                         else
3610                                 mlx5_hrxq_release(dev, dv->hrxq);
3611                         dv->hrxq = NULL;
3612                 }
3613         }
3614 }
3615
3616 /**
3617  * Remove the flow from the NIC and the memory.
3618  *
3619  * @param[in] dev
3620  *   Pointer to the Ethernet device structure.
3621  * @param[in, out] flow
3622  *   Pointer to flow structure.
3623  */
3624 static void
3625 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3626 {
3627         struct mlx5_flow *dev_flow;
3628
3629         if (!flow)
3630                 return;
3631         flow_dv_remove(dev, flow);
3632         if (flow->counter) {
3633                 flow_dv_counter_release(flow->counter);
3634                 flow->counter = NULL;
3635         }
3636         if (flow->tag_resource) {
3637                 flow_dv_tag_release(dev, flow->tag_resource);
3638                 flow->tag_resource = NULL;
3639         }
3640         while (!LIST_EMPTY(&flow->dev_flows)) {
3641                 dev_flow = LIST_FIRST(&flow->dev_flows);
3642                 LIST_REMOVE(dev_flow, next);
3643                 if (dev_flow->dv.matcher)
3644                         flow_dv_matcher_release(dev, dev_flow);
3645                 if (dev_flow->dv.encap_decap)
3646                         flow_dv_encap_decap_resource_release(dev_flow);
3647                 if (dev_flow->dv.modify_hdr)
3648                         flow_dv_modify_hdr_resource_release(dev_flow);
3649                 rte_free(dev_flow);
3650         }
3651 }
3652
3653 /**
3654  * Query a dv flow  rule for its statistics via devx.
3655  *
3656  * @param[in] dev
3657  *   Pointer to Ethernet device.
3658  * @param[in] flow
3659  *   Pointer to the sub flow.
3660  * @param[out] data
3661  *   data retrieved by the query.
3662  * @param[out] error
3663  *   Perform verbose error reporting if not NULL.
3664  *
3665  * @return
3666  *   0 on success, a negative errno value otherwise and rte_errno is set.
3667  */
3668 static int
3669 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
3670                     void *data, struct rte_flow_error *error)
3671 {
3672         struct mlx5_priv *priv = dev->data->dev_private;
3673         struct rte_flow_query_count *qc = data;
3674         uint64_t pkts = 0;
3675         uint64_t bytes = 0;
3676         int err;
3677
3678         if (!priv->config.devx)
3679                 return rte_flow_error_set(error, ENOTSUP,
3680                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3681                                           NULL,
3682                                           "counters are not supported");
3683         if (flow->counter) {
3684                 err = mlx5_devx_cmd_flow_counter_query
3685                                                 (flow->counter->dcs,
3686                                                  qc->reset, &pkts, &bytes);
3687                 if (err)
3688                         return rte_flow_error_set
3689                                 (error, err,
3690                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3691                                  NULL,
3692                                  "cannot read counters");
3693                 qc->hits_set = 1;
3694                 qc->bytes_set = 1;
3695                 qc->hits = pkts - flow->counter->hits;
3696                 qc->bytes = bytes - flow->counter->bytes;
3697                 if (qc->reset) {
3698                         flow->counter->hits = pkts;
3699                         flow->counter->bytes = bytes;
3700                 }
3701                 return 0;
3702         }
3703         return rte_flow_error_set(error, EINVAL,
3704                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3705                                   NULL,
3706                                   "counters are not available");
3707 }
3708
3709 /**
3710  * Query a flow.
3711  *
3712  * @see rte_flow_query()
3713  * @see rte_flow_ops
3714  */
3715 static int
3716 flow_dv_query(struct rte_eth_dev *dev,
3717               struct rte_flow *flow __rte_unused,
3718               const struct rte_flow_action *actions __rte_unused,
3719               void *data __rte_unused,
3720               struct rte_flow_error *error __rte_unused)
3721 {
3722         int ret = -EINVAL;
3723
3724         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3725                 switch (actions->type) {
3726                 case RTE_FLOW_ACTION_TYPE_VOID:
3727                         break;
3728                 case RTE_FLOW_ACTION_TYPE_COUNT:
3729                         ret = flow_dv_query_count(dev, flow, data, error);
3730                         break;
3731                 default:
3732                         return rte_flow_error_set(error, ENOTSUP,
3733                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3734                                                   actions,
3735                                                   "action not supported");
3736                 }
3737         }
3738         return ret;
3739 }
3740
3741
3742 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
3743         .validate = flow_dv_validate,
3744         .prepare = flow_dv_prepare,
3745         .translate = flow_dv_translate,
3746         .apply = flow_dv_apply,
3747         .remove = flow_dv_remove,
3748         .destroy = flow_dv_destroy,
3749         .query = flow_dv_query,
3750 };
3751
3752 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */