drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic error "-Wpedantic"
  18 #endif
  19
  20 #include <rte_common.h>
  21 #include <rte_ether.h>
  22 #include <rte_eth_ctrl.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28 #include <rte_gre.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  37
  38 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  39 #define MLX5DV_FLOW_ACTION_COUNTER_DEVX 0
  40 #endif
  41
  42 union flow_dv_attr {
  43         struct {
  44                 uint32_t valid:1;
  45                 uint32_t ipv4:1;
  46                 uint32_t ipv6:1;
  47                 uint32_t tcp:1;
  48                 uint32_t udp:1;
  49                 uint32_t reserved:27;
  50         };
  51         uint32_t attr;
  52 };
  53
  54 /**
  55  * Initialize flow attributes structure according to flow items' types.
  56  *
  57  * @param[in] item
  58  *   Pointer to item specification.
  59  * @param[out] attr
  60  *   Pointer to flow attributes structure.
  61  */
  62 static void
  63 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
  64 {
  65         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
  66                 switch (item->type) {
  67                 case RTE_FLOW_ITEM_TYPE_IPV4:
  68                         attr->ipv4 = 1;
  69                         break;
  70                 case RTE_FLOW_ITEM_TYPE_IPV6:
  71                         attr->ipv6 = 1;
  72                         break;
  73                 case RTE_FLOW_ITEM_TYPE_UDP:
  74                         attr->udp = 1;
  75                         break;
  76                 case RTE_FLOW_ITEM_TYPE_TCP:
  77                         attr->tcp = 1;
  78                         break;
  79                 default:
  80                         break;
  81                 }
  82         }
  83         attr->valid = 1;
  84 }
  85
  86 struct field_modify_info {
  87         uint32_t size; /* Size of field in protocol header, in bytes. */
  88         uint32_t offset; /* Offset of field in protocol header, in bytes. */
  89         enum mlx5_modification_field id;
  90 };
  91
  92 struct field_modify_info modify_eth[] = {
  93         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
  94         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
  95         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
  96         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
  97         {0, 0, 0},
  98 };
  99
 100 struct field_modify_info modify_ipv4[] = {
 101         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 102         {4, 12, MLX5_MODI_OUT_SIPV4},
 103         {4, 16, MLX5_MODI_OUT_DIPV4},
 104         {0, 0, 0},
 105 };
 106
 107 struct field_modify_info modify_ipv6[] = {
 108         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 109         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 110         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 111         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 112         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 113         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 114         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 115         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 116         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 117         {0, 0, 0},
 118 };
 119
 120 struct field_modify_info modify_udp[] = {
 121         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 122         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 123         {0, 0, 0},
 124 };
 125
 126 struct field_modify_info modify_tcp[] = {
 127         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 128         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 129         {0, 0, 0},
 130 };
 131
 132 /**
 133  * Convert modify-header action to DV specification.
 134  *
 135  * @param[in] item
 136  *   Pointer to item specification.
 137  * @param[in] field
 138  *   Pointer to field modification information.
 139  * @param[in,out] resource
 140  *   Pointer to the modify-header resource.
 141  * @param[in] type
 142  *   Type of modification.
 143  * @param[out] error
 144  *   Pointer to the error structure.
 145  *
 146  * @return
 147  *   0 on success, a negative errno value otherwise and rte_errno is set.
 148  */
 149 static int
 150 flow_dv_convert_modify_action(struct rte_flow_item *item,
 151                               struct field_modify_info *field,
 152                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 153                               uint32_t type,
 154                               struct rte_flow_error *error)
 155 {
 156         uint32_t i = resource->actions_num;
 157         struct mlx5_modification_cmd *actions = resource->actions;
 158         const uint8_t *spec = item->spec;
 159         const uint8_t *mask = item->mask;
 160         uint32_t set;
 161
 162         while (field->size) {
 163                 set = 0;
 164                 /* Generate modify command for each mask segment. */
 165                 memcpy(&set, &mask[field->offset], field->size);
 166                 if (set) {
 167                         if (i >= MLX5_MODIFY_NUM)
 168                                 return rte_flow_error_set(error, EINVAL,
 169                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 170                                          "too many items to modify");
 171                         actions[i].action_type = type;
 172                         actions[i].field = field->id;
 173                         actions[i].length = field->size ==
 174                                         4 ? 0 : field->size * 8;
 175                         rte_memcpy(&actions[i].data[4 - field->size],
 176                                    &spec[field->offset], field->size);
 177                         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 178                         ++i;
 179                 }
 180                 if (resource->actions_num != i)
 181                         resource->actions_num = i;
 182                 field++;
 183         }
 184         if (!resource->actions_num)
 185                 return rte_flow_error_set(error, EINVAL,
 186                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 187                                           "invalid modification flow item");
 188         return 0;
 189 }
 190
 191 /**
 192  * Convert modify-header set IPv4 address action to DV specification.
 193  *
 194  * @param[in,out] resource
 195  *   Pointer to the modify-header resource.
 196  * @param[in] action
 197  *   Pointer to action specification.
 198  * @param[out] error
 199  *   Pointer to the error structure.
 200  *
 201  * @return
 202  *   0 on success, a negative errno value otherwise and rte_errno is set.
 203  */
 204 static int
 205 flow_dv_convert_action_modify_ipv4
 206                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 207                          const struct rte_flow_action *action,
 208                          struct rte_flow_error *error)
 209 {
 210         const struct rte_flow_action_set_ipv4 *conf =
 211                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 212         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 213         struct rte_flow_item_ipv4 ipv4;
 214         struct rte_flow_item_ipv4 ipv4_mask;
 215
 216         memset(&ipv4, 0, sizeof(ipv4));
 217         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 218         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 219                 ipv4.hdr.src_addr = conf->ipv4_addr;
 220                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 221         } else {
 222                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 223                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 224         }
 225         item.spec = &ipv4;
 226         item.mask = &ipv4_mask;
 227         return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
 228                                              MLX5_MODIFICATION_TYPE_SET, error);
 229 }
 230
 231 /**
 232  * Convert modify-header set IPv6 address action to DV specification.
 233  *
 234  * @param[in,out] resource
 235  *   Pointer to the modify-header resource.
 236  * @param[in] action
 237  *   Pointer to action specification.
 238  * @param[out] error
 239  *   Pointer to the error structure.
 240  *
 241  * @return
 242  *   0 on success, a negative errno value otherwise and rte_errno is set.
 243  */
 244 static int
 245 flow_dv_convert_action_modify_ipv6
 246                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 247                          const struct rte_flow_action *action,
 248                          struct rte_flow_error *error)
 249 {
 250         const struct rte_flow_action_set_ipv6 *conf =
 251                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 252         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 253         struct rte_flow_item_ipv6 ipv6;
 254         struct rte_flow_item_ipv6 ipv6_mask;
 255
 256         memset(&ipv6, 0, sizeof(ipv6));
 257         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 258         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 259                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 260                        sizeof(ipv6.hdr.src_addr));
 261                 memcpy(&ipv6_mask.hdr.src_addr,
 262                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 263                        sizeof(ipv6.hdr.src_addr));
 264         } else {
 265                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 266                        sizeof(ipv6.hdr.dst_addr));
 267                 memcpy(&ipv6_mask.hdr.dst_addr,
 268                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 269                        sizeof(ipv6.hdr.dst_addr));
 270         }
 271         item.spec = &ipv6;
 272         item.mask = &ipv6_mask;
 273         return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
 274                                              MLX5_MODIFICATION_TYPE_SET, error);
 275 }
 276
 277 /**
 278  * Convert modify-header set MAC address action to DV specification.
 279  *
 280  * @param[in,out] resource
 281  *   Pointer to the modify-header resource.
 282  * @param[in] action
 283  *   Pointer to action specification.
 284  * @param[out] error
 285  *   Pointer to the error structure.
 286  *
 287  * @return
 288  *   0 on success, a negative errno value otherwise and rte_errno is set.
 289  */
 290 static int
 291 flow_dv_convert_action_modify_mac
 292                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 293                          const struct rte_flow_action *action,
 294                          struct rte_flow_error *error)
 295 {
 296         const struct rte_flow_action_set_mac *conf =
 297                 (const struct rte_flow_action_set_mac *)(action->conf);
 298         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 299         struct rte_flow_item_eth eth;
 300         struct rte_flow_item_eth eth_mask;
 301
 302         memset(&eth, 0, sizeof(eth));
 303         memset(&eth_mask, 0, sizeof(eth_mask));
 304         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 305                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 306                        sizeof(eth.src.addr_bytes));
 307                 memcpy(&eth_mask.src.addr_bytes,
 308                        &rte_flow_item_eth_mask.src.addr_bytes,
 309                        sizeof(eth_mask.src.addr_bytes));
 310         } else {
 311                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 312                        sizeof(eth.dst.addr_bytes));
 313                 memcpy(&eth_mask.dst.addr_bytes,
 314                        &rte_flow_item_eth_mask.dst.addr_bytes,
 315                        sizeof(eth_mask.dst.addr_bytes));
 316         }
 317         item.spec = &eth;
 318         item.mask = &eth_mask;
 319         return flow_dv_convert_modify_action(&item, modify_eth, resource,
 320                                              MLX5_MODIFICATION_TYPE_SET, error);
 321 }
 322
 323 /**
 324  * Convert modify-header set TP action to DV specification.
 325  *
 326  * @param[in,out] resource
 327  *   Pointer to the modify-header resource.
 328  * @param[in] action
 329  *   Pointer to action specification.
 330  * @param[in] items
 331  *   Pointer to rte_flow_item objects list.
 332  * @param[in] attr
 333  *   Pointer to flow attributes structure.
 334  * @param[out] error
 335  *   Pointer to the error structure.
 336  *
 337  * @return
 338  *   0 on success, a negative errno value otherwise and rte_errno is set.
 339  */
 340 static int
 341 flow_dv_convert_action_modify_tp
 342                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 343                          const struct rte_flow_action *action,
 344                          const struct rte_flow_item *items,
 345                          union flow_dv_attr *attr,
 346                          struct rte_flow_error *error)
 347 {
 348         const struct rte_flow_action_set_tp *conf =
 349                 (const struct rte_flow_action_set_tp *)(action->conf);
 350         struct rte_flow_item item;
 351         struct rte_flow_item_udp udp;
 352         struct rte_flow_item_udp udp_mask;
 353         struct rte_flow_item_tcp tcp;
 354         struct rte_flow_item_tcp tcp_mask;
 355         struct field_modify_info *field;
 356
 357         if (!attr->valid)
 358                 flow_dv_attr_init(items, attr);
 359         if (attr->udp) {
 360                 memset(&udp, 0, sizeof(udp));
 361                 memset(&udp_mask, 0, sizeof(udp_mask));
 362                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 363                         udp.hdr.src_port = conf->port;
 364                         udp_mask.hdr.src_port =
 365                                         rte_flow_item_udp_mask.hdr.src_port;
 366                 } else {
 367                         udp.hdr.dst_port = conf->port;
 368                         udp_mask.hdr.dst_port =
 369                                         rte_flow_item_udp_mask.hdr.dst_port;
 370                 }
 371                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 372                 item.spec = &udp;
 373                 item.mask = &udp_mask;
 374                 field = modify_udp;
 375         }
 376         if (attr->tcp) {
 377                 memset(&tcp, 0, sizeof(tcp));
 378                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 379                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 380                         tcp.hdr.src_port = conf->port;
 381                         tcp_mask.hdr.src_port =
 382                                         rte_flow_item_tcp_mask.hdr.src_port;
 383                 } else {
 384                         tcp.hdr.dst_port = conf->port;
 385                         tcp_mask.hdr.dst_port =
 386                                         rte_flow_item_tcp_mask.hdr.dst_port;
 387                 }
 388                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 389                 item.spec = &tcp;
 390                 item.mask = &tcp_mask;
 391                 field = modify_tcp;
 392         }
 393         return flow_dv_convert_modify_action(&item, field, resource,
 394                                              MLX5_MODIFICATION_TYPE_SET, error);
 395 }
 396
 397 /**
 398  * Convert modify-header set TTL action to DV specification.
 399  *
 400  * @param[in,out] resource
 401  *   Pointer to the modify-header resource.
 402  * @param[in] action
 403  *   Pointer to action specification.
 404  * @param[in] items
 405  *   Pointer to rte_flow_item objects list.
 406  * @param[in] attr
 407  *   Pointer to flow attributes structure.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_action_modify_ttl
 416                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 417                          const struct rte_flow_action *action,
 418                          const struct rte_flow_item *items,
 419                          union flow_dv_attr *attr,
 420                          struct rte_flow_error *error)
 421 {
 422         const struct rte_flow_action_set_ttl *conf =
 423                 (const struct rte_flow_action_set_ttl *)(action->conf);
 424         struct rte_flow_item item;
 425         struct rte_flow_item_ipv4 ipv4;
 426         struct rte_flow_item_ipv4 ipv4_mask;
 427         struct rte_flow_item_ipv6 ipv6;
 428         struct rte_flow_item_ipv6 ipv6_mask;
 429         struct field_modify_info *field;
 430
 431         if (!attr->valid)
 432                 flow_dv_attr_init(items, attr);
 433         if (attr->ipv4) {
 434                 memset(&ipv4, 0, sizeof(ipv4));
 435                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 436                 ipv4.hdr.time_to_live = conf->ttl_value;
 437                 ipv4_mask.hdr.time_to_live = 0xFF;
 438                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 439                 item.spec = &ipv4;
 440                 item.mask = &ipv4_mask;
 441                 field = modify_ipv4;
 442         }
 443         if (attr->ipv6) {
 444                 memset(&ipv6, 0, sizeof(ipv6));
 445                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 446                 ipv6.hdr.hop_limits = conf->ttl_value;
 447                 ipv6_mask.hdr.hop_limits = 0xFF;
 448                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 449                 item.spec = &ipv6;
 450                 item.mask = &ipv6_mask;
 451                 field = modify_ipv6;
 452         }
 453         return flow_dv_convert_modify_action(&item, field, resource,
 454                                              MLX5_MODIFICATION_TYPE_SET, error);
 455 }
 456
 457 /**
 458  * Convert modify-header decrement TTL action to DV specification.
 459  *
 460  * @param[in,out] resource
 461  *   Pointer to the modify-header resource.
 462  * @param[in] action
 463  *   Pointer to action specification.
 464  * @param[in] items
 465  *   Pointer to rte_flow_item objects list.
 466  * @param[in] attr
 467  *   Pointer to flow attributes structure.
 468  * @param[out] error
 469  *   Pointer to the error structure.
 470  *
 471  * @return
 472  *   0 on success, a negative errno value otherwise and rte_errno is set.
 473  */
 474 static int
 475 flow_dv_convert_action_modify_dec_ttl
 476                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 477                          const struct rte_flow_item *items,
 478                          union flow_dv_attr *attr,
 479                          struct rte_flow_error *error)
 480 {
 481         struct rte_flow_item item;
 482         struct rte_flow_item_ipv4 ipv4;
 483         struct rte_flow_item_ipv4 ipv4_mask;
 484         struct rte_flow_item_ipv6 ipv6;
 485         struct rte_flow_item_ipv6 ipv6_mask;
 486         struct field_modify_info *field;
 487
 488         if (!attr->valid)
 489                 flow_dv_attr_init(items, attr);
 490         if (attr->ipv4) {
 491                 memset(&ipv4, 0, sizeof(ipv4));
 492                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 493                 ipv4.hdr.time_to_live = 0xFF;
 494                 ipv4_mask.hdr.time_to_live = 0xFF;
 495                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 496                 item.spec = &ipv4;
 497                 item.mask = &ipv4_mask;
 498                 field = modify_ipv4;
 499         }
 500         if (attr->ipv6) {
 501                 memset(&ipv6, 0, sizeof(ipv6));
 502                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 503                 ipv6.hdr.hop_limits = 0xFF;
 504                 ipv6_mask.hdr.hop_limits = 0xFF;
 505                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 506                 item.spec = &ipv6;
 507                 item.mask = &ipv6_mask;
 508                 field = modify_ipv6;
 509         }
 510         return flow_dv_convert_modify_action(&item, field, resource,
 511                                              MLX5_MODIFICATION_TYPE_ADD, error);
 512 }
 513
 514 /**
 515  * Validate META item.
 516  *
 517  * @param[in] dev
 518  *   Pointer to the rte_eth_dev structure.
 519  * @param[in] item
 520  *   Item specification.
 521  * @param[in] attr
 522  *   Attributes of flow that includes this item.
 523  * @param[out] error
 524  *   Pointer to error structure.
 525  *
 526  * @return
 527  *   0 on success, a negative errno value otherwise and rte_errno is set.
 528  */
 529 static int
 530 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
 531                            const struct rte_flow_item *item,
 532                            const struct rte_flow_attr *attr,
 533                            struct rte_flow_error *error)
 534 {
 535         const struct rte_flow_item_meta *spec = item->spec;
 536         const struct rte_flow_item_meta *mask = item->mask;
 537         const struct rte_flow_item_meta nic_mask = {
 538                 .data = RTE_BE32(UINT32_MAX)
 539         };
 540         int ret;
 541         uint64_t offloads = dev->data->dev_conf.txmode.offloads;
 542
 543         if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
 544                 return rte_flow_error_set(error, EPERM,
 545                                           RTE_FLOW_ERROR_TYPE_ITEM,
 546                                           NULL,
 547                                           "match on metadata offload "
 548                                           "configuration is off for this port");
 549         if (!spec)
 550                 return rte_flow_error_set(error, EINVAL,
 551                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 552                                           item->spec,
 553                                           "data cannot be empty");
 554         if (!spec->data)
 555                 return rte_flow_error_set(error, EINVAL,
 556                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 557                                           NULL,
 558                                           "data cannot be zero");
 559         if (!mask)
 560                 mask = &rte_flow_item_meta_mask;
 561         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 562                                         (const uint8_t *)&nic_mask,
 563                                         sizeof(struct rte_flow_item_meta),
 564                                         error);
 565         if (ret < 0)
 566                 return ret;
 567         if (attr->ingress)
 568                 return rte_flow_error_set(error, ENOTSUP,
 569                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 570                                           NULL,
 571                                           "pattern not supported for ingress");
 572         return 0;
 573 }
 574
 575 /**
 576  * Validate count action.
 577  *
 578  * @param[in] dev
 579  *   device otr.
 580  * @param[out] error
 581  *   Pointer to error structure.
 582  *
 583  * @return
 584  *   0 on success, a negative errno value otherwise and rte_errno is set.
 585  */
 586 static int
 587 flow_dv_validate_action_count(struct rte_eth_dev *dev,
 588                               struct rte_flow_error *error)
 589 {
 590         struct priv *priv = dev->data->dev_private;
 591
 592         if (!priv->config.devx)
 593                 goto notsup_err;
 594 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
 595         return 0;
 596 #endif
 597 notsup_err:
 598         return rte_flow_error_set
 599                       (error, ENOTSUP,
 600                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 601                        NULL,
 602                        "count action not supported");
 603 }
 604
 605 /**
 606  * Validate the L2 encap action.
 607  *
 608  * @param[in] action_flags
 609  *   Holds the actions detected until now.
 610  * @param[in] action
 611  *   Pointer to the encap action.
 612  * @param[in] attr
 613  *   Pointer to flow attributes
 614  * @param[out] error
 615  *   Pointer to error structure.
 616  *
 617  * @return
 618  *   0 on success, a negative errno value otherwise and rte_errno is set.
 619  */
 620 static int
 621 flow_dv_validate_action_l2_encap(uint64_t action_flags,
 622                                  const struct rte_flow_action *action,
 623                                  const struct rte_flow_attr *attr,
 624                                  struct rte_flow_error *error)
 625 {
 626         if (!(action->conf))
 627                 return rte_flow_error_set(error, EINVAL,
 628                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 629                                           "configuration cannot be null");
 630         if (action_flags & MLX5_FLOW_ACTION_DROP)
 631                 return rte_flow_error_set(error, EINVAL,
 632                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 633                                           "can't drop and encap in same flow");
 634         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 635                 return rte_flow_error_set(error, EINVAL,
 636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 637                                           "can only have a single encap or"
 638                                           " decap action in a flow");
 639         if (attr->ingress)
 640                 return rte_flow_error_set(error, ENOTSUP,
 641                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 642                                           NULL,
 643                                           "encap action not supported for "
 644                                           "ingress");
 645         return 0;
 646 }
 647
 648 /**
 649  * Validate the L2 decap action.
 650  *
 651  * @param[in] action_flags
 652  *   Holds the actions detected until now.
 653  * @param[in] attr
 654  *   Pointer to flow attributes
 655  * @param[out] error
 656  *   Pointer to error structure.
 657  *
 658  * @return
 659  *   0 on success, a negative errno value otherwise and rte_errno is set.
 660  */
 661 static int
 662 flow_dv_validate_action_l2_decap(uint64_t action_flags,
 663                                  const struct rte_flow_attr *attr,
 664                                  struct rte_flow_error *error)
 665 {
 666         if (action_flags & MLX5_FLOW_ACTION_DROP)
 667                 return rte_flow_error_set(error, EINVAL,
 668                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 669                                           "can't drop and decap in same flow");
 670         if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
 671                 return rte_flow_error_set(error, EINVAL,
 672                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 673                                           "can only have a single encap or"
 674                                           " decap action in a flow");
 675         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 676                 return rte_flow_error_set(error, EINVAL,
 677                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 678                                           "can't have decap action after"
 679                                           " modify action");
 680         if (attr->egress)
 681                 return rte_flow_error_set(error, ENOTSUP,
 682                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 683                                           NULL,
 684                                           "decap action not supported for "
 685                                           "egress");
 686         return 0;
 687 }
 688
 689 /**
 690  * Validate the raw encap action.
 691  *
 692  * @param[in] action_flags
 693  *   Holds the actions detected until now.
 694  * @param[in] action
 695  *   Pointer to the encap action.
 696  * @param[in] attr
 697  *   Pointer to flow attributes
 698  * @param[out] error
 699  *   Pointer to error structure.
 700  *
 701  * @return
 702  *   0 on success, a negative errno value otherwise and rte_errno is set.
 703  */
 704 static int
 705 flow_dv_validate_action_raw_encap(uint64_t action_flags,
 706                                   const struct rte_flow_action *action,
 707                                   const struct rte_flow_attr *attr,
 708                                   struct rte_flow_error *error)
 709 {
 710         if (!(action->conf))
 711                 return rte_flow_error_set(error, EINVAL,
 712                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
 713                                           "configuration cannot be null");
 714         if (action_flags & MLX5_FLOW_ACTION_DROP)
 715                 return rte_flow_error_set(error, EINVAL,
 716                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 717                                           "can't drop and encap in same flow");
 718         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 719                 return rte_flow_error_set(error, EINVAL,
 720                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 721                                           "can only have a single encap"
 722                                           " action in a flow");
 723         /* encap without preceding decap is not supported for ingress */
 724         if (attr->ingress && !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
 725                 return rte_flow_error_set(error, ENOTSUP,
 726                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 727                                           NULL,
 728                                           "encap action not supported for "
 729                                           "ingress");
 730         return 0;
 731 }
 732
 733 /**
 734  * Validate the raw decap action.
 735  *
 736  * @param[in] action_flags
 737  *   Holds the actions detected until now.
 738  * @param[in] action
 739  *   Pointer to the encap action.
 740  * @param[in] attr
 741  *   Pointer to flow attributes
 742  * @param[out] error
 743  *   Pointer to error structure.
 744  *
 745  * @return
 746  *   0 on success, a negative errno value otherwise and rte_errno is set.
 747  */
 748 static int
 749 flow_dv_validate_action_raw_decap(uint64_t action_flags,
 750                                   const struct rte_flow_action *action,
 751                                   const struct rte_flow_attr *attr,
 752                                   struct rte_flow_error *error)
 753 {
 754         if (action_flags & MLX5_FLOW_ACTION_DROP)
 755                 return rte_flow_error_set(error, EINVAL,
 756                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 757                                           "can't drop and decap in same flow");
 758         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
 759                 return rte_flow_error_set(error, EINVAL,
 760                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 761                                           "can't have encap action before"
 762                                           " decap action");
 763         if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
 764                 return rte_flow_error_set(error, EINVAL,
 765                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 766                                           "can only have a single decap"
 767                                           " action in a flow");
 768         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
 769                 return rte_flow_error_set(error, EINVAL,
 770                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 771                                           "can't have decap action after"
 772                                           " modify action");
 773         /* decap action is valid on egress only if it is followed by encap */
 774         if (attr->egress) {
 775                 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
 776                        action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
 777                        action++) {
 778                 }
 779                 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
 780                         return rte_flow_error_set
 781                                         (error, ENOTSUP,
 782                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 783                                          NULL, "decap action not supported"
 784                                          " for egress");
 785         }
 786         return 0;
 787 }
 788
 789 /**
 790  * Find existing encap/decap resource or create and register a new one.
 791  *
 792  * @param dev[in, out]
 793  *   Pointer to rte_eth_dev structure.
 794  * @param[in, out] resource
 795  *   Pointer to encap/decap resource.
 796  * @parm[in, out] dev_flow
 797  *   Pointer to the dev_flow.
 798  * @param[out] error
 799  *   pointer to error structure.
 800  *
 801  * @return
 802  *   0 on success otherwise -errno and errno is set.
 803  */
 804 static int
 805 flow_dv_encap_decap_resource_register
 806                         (struct rte_eth_dev *dev,
 807                          struct mlx5_flow_dv_encap_decap_resource *resource,
 808                          struct mlx5_flow *dev_flow,
 809                          struct rte_flow_error *error)
 810 {
 811         struct priv *priv = dev->data->dev_private;
 812         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
 813
 814         /* Lookup a matching resource from cache. */
 815         LIST_FOREACH(cache_resource, &priv->encaps_decaps, next) {
 816                 if (resource->reformat_type == cache_resource->reformat_type &&
 817                     resource->ft_type == cache_resource->ft_type &&
 818                     resource->size == cache_resource->size &&
 819                     !memcmp((const void *)resource->buf,
 820                             (const void *)cache_resource->buf,
 821                             resource->size)) {
 822                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
 823                                 (void *)cache_resource,
 824                                 rte_atomic32_read(&cache_resource->refcnt));
 825                         rte_atomic32_inc(&cache_resource->refcnt);
 826                         dev_flow->dv.encap_decap = cache_resource;
 827                         return 0;
 828                 }
 829         }
 830         /* Register new encap/decap resource. */
 831         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
 832         if (!cache_resource)
 833                 return rte_flow_error_set(error, ENOMEM,
 834                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 835                                           "cannot allocate resource memory");
 836         *cache_resource = *resource;
 837         cache_resource->verbs_action =
 838                 mlx5_glue->dv_create_flow_action_packet_reformat
 839                         (priv->ctx, cache_resource->size,
 840                          (cache_resource->size ? cache_resource->buf : NULL),
 841                          cache_resource->reformat_type,
 842                          cache_resource->ft_type);
 843         if (!cache_resource->verbs_action) {
 844                 rte_free(cache_resource);
 845                 return rte_flow_error_set(error, ENOMEM,
 846                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 847                                           NULL, "cannot create action");
 848         }
 849         rte_atomic32_init(&cache_resource->refcnt);
 850         rte_atomic32_inc(&cache_resource->refcnt);
 851         LIST_INSERT_HEAD(&priv->encaps_decaps, cache_resource, next);
 852         dev_flow->dv.encap_decap = cache_resource;
 853         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
 854                 (void *)cache_resource,
 855                 rte_atomic32_read(&cache_resource->refcnt));
 856         return 0;
 857 }
 858
 859 /**
 860  * Get the size of specific rte_flow_item_type
 861  *
 862  * @param[in] item_type
 863  *   Tested rte_flow_item_type.
 864  *
 865  * @return
 866  *   sizeof struct item_type, 0 if void or irrelevant.
 867  */
 868 static size_t
 869 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
 870 {
 871         size_t retval;
 872
 873         switch (item_type) {
 874         case RTE_FLOW_ITEM_TYPE_ETH:
 875                 retval = sizeof(struct rte_flow_item_eth);
 876                 break;
 877         case RTE_FLOW_ITEM_TYPE_VLAN:
 878                 retval = sizeof(struct rte_flow_item_vlan);
 879                 break;
 880         case RTE_FLOW_ITEM_TYPE_IPV4:
 881                 retval = sizeof(struct rte_flow_item_ipv4);
 882                 break;
 883         case RTE_FLOW_ITEM_TYPE_IPV6:
 884                 retval = sizeof(struct rte_flow_item_ipv6);
 885                 break;
 886         case RTE_FLOW_ITEM_TYPE_UDP:
 887                 retval = sizeof(struct rte_flow_item_udp);
 888                 break;
 889         case RTE_FLOW_ITEM_TYPE_TCP:
 890                 retval = sizeof(struct rte_flow_item_tcp);
 891                 break;
 892         case RTE_FLOW_ITEM_TYPE_VXLAN:
 893                 retval = sizeof(struct rte_flow_item_vxlan);
 894                 break;
 895         case RTE_FLOW_ITEM_TYPE_GRE:
 896                 retval = sizeof(struct rte_flow_item_gre);
 897                 break;
 898         case RTE_FLOW_ITEM_TYPE_NVGRE:
 899                 retval = sizeof(struct rte_flow_item_nvgre);
 900                 break;
 901         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 902                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
 903                 break;
 904         case RTE_FLOW_ITEM_TYPE_MPLS:
 905                 retval = sizeof(struct rte_flow_item_mpls);
 906                 break;
 907         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
 908         default:
 909                 retval = 0;
 910                 break;
 911         }
 912         return retval;
 913 }
 914
 915 #define MLX5_ENCAP_IPV4_VERSION         0x40
 916 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
 917 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
 918 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
 919 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
 920 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
 921 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
 922
 923 /**
 924  * Convert the encap action data from list of rte_flow_item to raw buffer
 925  *
 926  * @param[in] items
 927  *   Pointer to rte_flow_item objects list.
 928  * @param[out] buf
 929  *   Pointer to the output buffer.
 930  * @param[out] size
 931  *   Pointer to the output buffer size.
 932  * @param[out] error
 933  *   Pointer to the error structure.
 934  *
 935  * @return
 936  *   0 on success, a negative errno value otherwise and rte_errno is set.
 937  */
 938 static int
 939 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 940                            size_t *size, struct rte_flow_error *error)
 941 {
 942         struct ether_hdr *eth = NULL;
 943         struct vlan_hdr *vlan = NULL;
 944         struct ipv4_hdr *ipv4 = NULL;
 945         struct ipv6_hdr *ipv6 = NULL;
 946         struct udp_hdr *udp = NULL;
 947         struct vxlan_hdr *vxlan = NULL;
 948         struct vxlan_gpe_hdr *vxlan_gpe = NULL;
 949         struct gre_hdr *gre = NULL;
 950         size_t len;
 951         size_t temp_size = 0;
 952
 953         if (!items)
 954                 return rte_flow_error_set(error, EINVAL,
 955                                           RTE_FLOW_ERROR_TYPE_ACTION,
 956                                           NULL, "invalid empty data");
 957         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 958                 len = flow_dv_get_item_len(items->type);
 959                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
 960                         return rte_flow_error_set(error, EINVAL,
 961                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 962                                                   (void *)items->type,
 963                                                   "items total size is too big"
 964                                                   " for encap action");
 965                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
 966                 switch (items->type) {
 967                 case RTE_FLOW_ITEM_TYPE_ETH:
 968                         eth = (struct ether_hdr *)&buf[temp_size];
 969                         break;
 970                 case RTE_FLOW_ITEM_TYPE_VLAN:
 971                         vlan = (struct vlan_hdr *)&buf[temp_size];
 972                         if (!eth)
 973                                 return rte_flow_error_set(error, EINVAL,
 974                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 975                                                 (void *)items->type,
 976                                                 "eth header not found");
 977                         if (!eth->ether_type)
 978                                 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
 979                         break;
 980                 case RTE_FLOW_ITEM_TYPE_IPV4:
 981                         ipv4 = (struct ipv4_hdr *)&buf[temp_size];
 982                         if (!vlan && !eth)
 983                                 return rte_flow_error_set(error, EINVAL,
 984                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 985                                                 (void *)items->type,
 986                                                 "neither eth nor vlan"
 987                                                 " header found");
 988                         if (vlan && !vlan->eth_proto)
 989                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
 990                         else if (eth && !eth->ether_type)
 991                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
 992                         if (!ipv4->version_ihl)
 993                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
 994                                                     MLX5_ENCAP_IPV4_IHL_MIN;
 995                         if (!ipv4->time_to_live)
 996                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
 997                         break;
 998                 case RTE_FLOW_ITEM_TYPE_IPV6:
 999                         ipv6 = (struct ipv6_hdr *)&buf[temp_size];
1000                         if (!vlan && !eth)
1001                                 return rte_flow_error_set(error, EINVAL,
1002                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1003                                                 (void *)items->type,
1004                                                 "neither eth nor vlan"
1005                                                 " header found");
1006                         if (vlan && !vlan->eth_proto)
1007                                 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
1008                         else if (eth && !eth->ether_type)
1009                                 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
1010                         if (!ipv6->vtc_flow)
1011                                 ipv6->vtc_flow =
1012                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1013                         if (!ipv6->hop_limits)
1014                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1015                         break;
1016                 case RTE_FLOW_ITEM_TYPE_UDP:
1017                         udp = (struct udp_hdr *)&buf[temp_size];
1018                         if (!ipv4 && !ipv6)
1019                                 return rte_flow_error_set(error, EINVAL,
1020                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1021                                                 (void *)items->type,
1022                                                 "ip header not found");
1023                         if (ipv4 && !ipv4->next_proto_id)
1024                                 ipv4->next_proto_id = IPPROTO_UDP;
1025                         else if (ipv6 && !ipv6->proto)
1026                                 ipv6->proto = IPPROTO_UDP;
1027                         break;
1028                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1029                         vxlan = (struct vxlan_hdr *)&buf[temp_size];
1030                         if (!udp)
1031                                 return rte_flow_error_set(error, EINVAL,
1032                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1033                                                 (void *)items->type,
1034                                                 "udp header not found");
1035                         if (!udp->dst_port)
1036                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1037                         if (!vxlan->vx_flags)
1038                                 vxlan->vx_flags =
1039                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1040                         break;
1041                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1042                         vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
1043                         if (!udp)
1044                                 return rte_flow_error_set(error, EINVAL,
1045                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1046                                                 (void *)items->type,
1047                                                 "udp header not found");
1048                         if (!vxlan_gpe->proto)
1049                                 return rte_flow_error_set(error, EINVAL,
1050                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1051                                                 (void *)items->type,
1052                                                 "next protocol not found");
1053                         if (!udp->dst_port)
1054                                 udp->dst_port =
1055                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1056                         if (!vxlan_gpe->vx_flags)
1057                                 vxlan_gpe->vx_flags =
1058                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1059                         break;
1060                 case RTE_FLOW_ITEM_TYPE_GRE:
1061                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1062                         gre = (struct gre_hdr *)&buf[temp_size];
1063                         if (!gre->proto)
1064                                 return rte_flow_error_set(error, EINVAL,
1065                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1066                                                 (void *)items->type,
1067                                                 "next protocol not found");
1068                         if (!ipv4 && !ipv6)
1069                                 return rte_flow_error_set(error, EINVAL,
1070                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1071                                                 (void *)items->type,
1072                                                 "ip header not found");
1073                         if (ipv4 && !ipv4->next_proto_id)
1074                                 ipv4->next_proto_id = IPPROTO_GRE;
1075                         else if (ipv6 && !ipv6->proto)
1076                                 ipv6->proto = IPPROTO_GRE;
1077                         break;
1078                 case RTE_FLOW_ITEM_TYPE_VOID:
1079                         break;
1080                 default:
1081                         return rte_flow_error_set(error, EINVAL,
1082                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1083                                                   (void *)items->type,
1084                                                   "unsupported item type");
1085                         break;
1086                 }
1087                 temp_size += len;
1088         }
1089         *size = temp_size;
1090         return 0;
1091 }
1092
1093 /**
1094  * Convert L2 encap action to DV specification.
1095  *
1096  * @param[in] dev
1097  *   Pointer to rte_eth_dev structure.
1098  * @param[in] action
1099  *   Pointer to action structure.
1100  * @param[in, out] dev_flow
1101  *   Pointer to the mlx5_flow.
1102  * @param[out] error
1103  *   Pointer to the error structure.
1104  *
1105  * @return
1106  *   0 on success, a negative errno value otherwise and rte_errno is set.
1107  */
1108 static int
1109 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1110                                const struct rte_flow_action *action,
1111                                struct mlx5_flow *dev_flow,
1112                                struct rte_flow_error *error)
1113 {
1114         const struct rte_flow_item *encap_data;
1115         const struct rte_flow_action_raw_encap *raw_encap_data;
1116         struct mlx5_flow_dv_encap_decap_resource res = {
1117                 .reformat_type =
1118                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1119                 .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1120         };
1121
1122         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1123                 raw_encap_data =
1124                         (const struct rte_flow_action_raw_encap *)action->conf;
1125                 res.size = raw_encap_data->size;
1126                 memcpy(res.buf, raw_encap_data->data, res.size);
1127         } else {
1128                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1129                         encap_data =
1130                                 ((const struct rte_flow_action_vxlan_encap *)
1131                                                 action->conf)->definition;
1132                 else
1133                         encap_data =
1134                                 ((const struct rte_flow_action_nvgre_encap *)
1135                                                 action->conf)->definition;
1136                 if (flow_dv_convert_encap_data(encap_data, res.buf,
1137                                                &res.size, error))
1138                         return -rte_errno;
1139         }
1140         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1141                 return rte_flow_error_set(error, EINVAL,
1142                                           RTE_FLOW_ERROR_TYPE_ACTION,
1143                                           NULL, "can't create L2 encap action");
1144         return 0;
1145 }
1146
1147 /**
1148  * Convert L2 decap action to DV specification.
1149  *
1150  * @param[in] dev
1151  *   Pointer to rte_eth_dev structure.
1152  * @param[in, out] dev_flow
1153  *   Pointer to the mlx5_flow.
1154  * @param[out] error
1155  *   Pointer to the error structure.
1156  *
1157  * @return
1158  *   0 on success, a negative errno value otherwise and rte_errno is set.
1159  */
1160 static int
1161 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1162                                struct mlx5_flow *dev_flow,
1163                                struct rte_flow_error *error)
1164 {
1165         struct mlx5_flow_dv_encap_decap_resource res = {
1166                 .size = 0,
1167                 .reformat_type =
1168                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1169                 .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1170         };
1171
1172         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1173                 return rte_flow_error_set(error, EINVAL,
1174                                           RTE_FLOW_ERROR_TYPE_ACTION,
1175                                           NULL, "can't create L2 decap action");
1176         return 0;
1177 }
1178
1179 /**
1180  * Convert raw decap/encap (L3 tunnel) action to DV specification.
1181  *
1182  * @param[in] dev
1183  *   Pointer to rte_eth_dev structure.
1184  * @param[in] action
1185  *   Pointer to action structure.
1186  * @param[in, out] dev_flow
1187  *   Pointer to the mlx5_flow.
1188  * @param[in] attr
1189  *   Pointer to the flow attributes.
1190  * @param[out] error
1191  *   Pointer to the error structure.
1192  *
1193  * @return
1194  *   0 on success, a negative errno value otherwise and rte_errno is set.
1195  */
1196 static int
1197 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1198                                 const struct rte_flow_action *action,
1199                                 struct mlx5_flow *dev_flow,
1200                                 const struct rte_flow_attr *attr,
1201                                 struct rte_flow_error *error)
1202 {
1203         const struct rte_flow_action_raw_encap *encap_data;
1204         struct mlx5_flow_dv_encap_decap_resource res;
1205
1206         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1207         res.size = encap_data->size;
1208         memcpy(res.buf, encap_data->data, res.size);
1209         res.reformat_type = attr->egress ?
1210                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1211                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1212         res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1213                                      MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1214         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1215                 return rte_flow_error_set(error, EINVAL,
1216                                           RTE_FLOW_ERROR_TYPE_ACTION,
1217                                           NULL, "can't create encap action");
1218         return 0;
1219 }
1220
1221 /**
1222  * Validate the modify-header actions.
1223  *
1224  * @param[in] action_flags
1225  *   Holds the actions detected until now.
1226  * @param[in] action
1227  *   Pointer to the modify action.
1228  * @param[out] error
1229  *   Pointer to error structure.
1230  *
1231  * @return
1232  *   0 on success, a negative errno value otherwise and rte_errno is set.
1233  */
1234 static int
1235 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1236                                    const struct rte_flow_action *action,
1237                                    struct rte_flow_error *error)
1238 {
1239         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1240                 return rte_flow_error_set(error, EINVAL,
1241                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1242                                           NULL, "action configuration not set");
1243         if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1244                 return rte_flow_error_set(error, EINVAL,
1245                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1246                                           "can't have encap action before"
1247                                           " modify action");
1248         return 0;
1249 }
1250
1251 /**
1252  * Validate the modify-header MAC address actions.
1253  *
1254  * @param[in] action_flags
1255  *   Holds the actions detected until now.
1256  * @param[in] action
1257  *   Pointer to the modify action.
1258  * @param[in] item_flags
1259  *   Holds the items detected.
1260  * @param[out] error
1261  *   Pointer to error structure.
1262  *
1263  * @return
1264  *   0 on success, a negative errno value otherwise and rte_errno is set.
1265  */
1266 static int
1267 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1268                                    const struct rte_flow_action *action,
1269                                    const uint64_t item_flags,
1270                                    struct rte_flow_error *error)
1271 {
1272         int ret = 0;
1273
1274         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1275         if (!ret) {
1276                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1277                         return rte_flow_error_set(error, EINVAL,
1278                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1279                                                   NULL,
1280                                                   "no L2 item in pattern");
1281         }
1282         return ret;
1283 }
1284
1285 /**
1286  * Validate the modify-header IPv4 address actions.
1287  *
1288  * @param[in] action_flags
1289  *   Holds the actions detected until now.
1290  * @param[in] action
1291  *   Pointer to the modify action.
1292  * @param[in] item_flags
1293  *   Holds the items detected.
1294  * @param[out] error
1295  *   Pointer to error structure.
1296  *
1297  * @return
1298  *   0 on success, a negative errno value otherwise and rte_errno is set.
1299  */
1300 static int
1301 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
1302                                     const struct rte_flow_action *action,
1303                                     const uint64_t item_flags,
1304                                     struct rte_flow_error *error)
1305 {
1306         int ret = 0;
1307
1308         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1309         if (!ret) {
1310                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
1311                         return rte_flow_error_set(error, EINVAL,
1312                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1313                                                   NULL,
1314                                                   "no ipv4 item in pattern");
1315         }
1316         return ret;
1317 }
1318
1319 /**
1320  * Validate the modify-header IPv6 address actions.
1321  *
1322  * @param[in] action_flags
1323  *   Holds the actions detected until now.
1324  * @param[in] action
1325  *   Pointer to the modify action.
1326  * @param[in] item_flags
1327  *   Holds the items detected.
1328  * @param[out] error
1329  *   Pointer to error structure.
1330  *
1331  * @return
1332  *   0 on success, a negative errno value otherwise and rte_errno is set.
1333  */
1334 static int
1335 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
1336                                     const struct rte_flow_action *action,
1337                                     const uint64_t item_flags,
1338                                     struct rte_flow_error *error)
1339 {
1340         int ret = 0;
1341
1342         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1343         if (!ret) {
1344                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
1345                         return rte_flow_error_set(error, EINVAL,
1346                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1347                                                   NULL,
1348                                                   "no ipv6 item in pattern");
1349         }
1350         return ret;
1351 }
1352
1353 /**
1354  * Validate the modify-header TP actions.
1355  *
1356  * @param[in] action_flags
1357  *   Holds the actions detected until now.
1358  * @param[in] action
1359  *   Pointer to the modify action.
1360  * @param[in] item_flags
1361  *   Holds the items detected.
1362  * @param[out] error
1363  *   Pointer to error structure.
1364  *
1365  * @return
1366  *   0 on success, a negative errno value otherwise and rte_errno is set.
1367  */
1368 static int
1369 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
1370                                   const struct rte_flow_action *action,
1371                                   const uint64_t item_flags,
1372                                   struct rte_flow_error *error)
1373 {
1374         int ret = 0;
1375
1376         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1377         if (!ret) {
1378                 if (!(item_flags & MLX5_FLOW_LAYER_L4))
1379                         return rte_flow_error_set(error, EINVAL,
1380                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1381                                                   NULL, "no transport layer "
1382                                                   "in pattern");
1383         }
1384         return ret;
1385 }
1386
1387 /**
1388  * Validate the modify-header TTL actions.
1389  *
1390  * @param[in] action_flags
1391  *   Holds the actions detected until now.
1392  * @param[in] action
1393  *   Pointer to the modify action.
1394  * @param[in] item_flags
1395  *   Holds the items detected.
1396  * @param[out] error
1397  *   Pointer to error structure.
1398  *
1399  * @return
1400  *   0 on success, a negative errno value otherwise and rte_errno is set.
1401  */
1402 static int
1403 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
1404                                    const struct rte_flow_action *action,
1405                                    const uint64_t item_flags,
1406                                    struct rte_flow_error *error)
1407 {
1408         int ret = 0;
1409
1410         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1411         if (!ret) {
1412                 if (!(item_flags & MLX5_FLOW_LAYER_L3))
1413                         return rte_flow_error_set(error, EINVAL,
1414                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1415                                                   NULL,
1416                                                   "no IP protocol in pattern");
1417         }
1418         return ret;
1419 }
1420
1421 /**
1422  * Find existing modify-header resource or create and register a new one.
1423  *
1424  * @param dev[in, out]
1425  *   Pointer to rte_eth_dev structure.
1426  * @param[in, out] resource
1427  *   Pointer to modify-header resource.
1428  * @parm[in, out] dev_flow
1429  *   Pointer to the dev_flow.
1430  * @param[out] error
1431  *   pointer to error structure.
1432  *
1433  * @return
1434  *   0 on success otherwise -errno and errno is set.
1435  */
1436 static int
1437 flow_dv_modify_hdr_resource_register
1438                         (struct rte_eth_dev *dev,
1439                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1440                          struct mlx5_flow *dev_flow,
1441                          struct rte_flow_error *error)
1442 {
1443         struct priv *priv = dev->data->dev_private;
1444         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
1445
1446         /* Lookup a matching resource from cache. */
1447         LIST_FOREACH(cache_resource, &priv->modify_cmds, next) {
1448                 if (resource->ft_type == cache_resource->ft_type &&
1449                     resource->actions_num == cache_resource->actions_num &&
1450                     !memcmp((const void *)resource->actions,
1451                             (const void *)cache_resource->actions,
1452                             (resource->actions_num *
1453                                             sizeof(resource->actions[0])))) {
1454                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
1455                                 (void *)cache_resource,
1456                                 rte_atomic32_read(&cache_resource->refcnt));
1457                         rte_atomic32_inc(&cache_resource->refcnt);
1458                         dev_flow->dv.modify_hdr = cache_resource;
1459                         return 0;
1460                 }
1461         }
1462         /* Register new modify-header resource. */
1463         cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1464         if (!cache_resource)
1465                 return rte_flow_error_set(error, ENOMEM,
1466                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1467                                           "cannot allocate resource memory");
1468         *cache_resource = *resource;
1469         cache_resource->verbs_action =
1470                 mlx5_glue->dv_create_flow_action_modify_header
1471                                         (priv->ctx,
1472                                          cache_resource->actions_num *
1473                                          sizeof(cache_resource->actions[0]),
1474                                          (uint64_t *)cache_resource->actions,
1475                                          cache_resource->ft_type);
1476         if (!cache_resource->verbs_action) {
1477                 rte_free(cache_resource);
1478                 return rte_flow_error_set(error, ENOMEM,
1479                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1480                                           NULL, "cannot create action");
1481         }
1482         rte_atomic32_init(&cache_resource->refcnt);
1483         rte_atomic32_inc(&cache_resource->refcnt);
1484         LIST_INSERT_HEAD(&priv->modify_cmds, cache_resource, next);
1485         dev_flow->dv.modify_hdr = cache_resource;
1486         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
1487                 (void *)cache_resource,
1488                 rte_atomic32_read(&cache_resource->refcnt));
1489         return 0;
1490 }
1491
1492 /**
1493  * Get or create a flow counter.
1494  *
1495  * @param[in] dev
1496  *   Pointer to the Ethernet device structure.
1497  * @param[in] shared
1498  *   Indicate if this counter is shared with other flows.
1499  * @param[in] id
1500  *   Counter identifier.
1501  *
1502  * @return
1503  *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
1504  */
1505 static struct mlx5_flow_counter *
1506 flow_dv_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
1507 {
1508         struct priv *priv = dev->data->dev_private;
1509         struct mlx5_flow_counter *cnt = NULL;
1510         struct mlx5_devx_counter_set *dcs = NULL;
1511         int ret;
1512
1513         if (!priv->config.devx) {
1514                 ret = -ENOTSUP;
1515                 goto error_exit;
1516         }
1517         if (shared) {
1518                 LIST_FOREACH(cnt, &priv->flow_counters, next) {
1519                         if (cnt->shared && cnt->id == id) {
1520                                 cnt->ref_cnt++;
1521                                 return cnt;
1522                         }
1523                 }
1524         }
1525         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
1526         dcs = rte_calloc(__func__, 1, sizeof(*dcs), 0);
1527         if (!dcs || !cnt) {
1528                 ret = -ENOMEM;
1529                 goto error_exit;
1530         }
1531         ret = mlx5_devx_cmd_flow_counter_alloc(priv->ctx, dcs);
1532         if (ret)
1533                 goto error_exit;
1534         struct mlx5_flow_counter tmpl = {
1535                 .shared = shared,
1536                 .ref_cnt = 1,
1537                 .id = id,
1538                 .dcs = dcs,
1539         };
1540         *cnt = tmpl;
1541         LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
1542         return cnt;
1543 error_exit:
1544         rte_free(cnt);
1545         rte_free(dcs);
1546         rte_errno = -ret;
1547         return NULL;
1548 }
1549
1550 /**
1551  * Release a flow counter.
1552  *
1553  * @param[in] counter
1554  *   Pointer to the counter handler.
1555  */
1556 static void
1557 flow_dv_counter_release(struct mlx5_flow_counter *counter)
1558 {
1559         int ret;
1560
1561         if (!counter)
1562                 return;
1563         if (--counter->ref_cnt == 0) {
1564                 ret = mlx5_devx_cmd_flow_counter_free(counter->dcs->obj);
1565                 if (ret)
1566                         DRV_LOG(ERR, "Failed to free devx counters, %d", ret);
1567                 LIST_REMOVE(counter, next);
1568                 rte_free(counter->dcs);
1569                 rte_free(counter);
1570         }
1571 }
1572
1573 /**
1574  * Verify the @p attributes will be correctly understood by the NIC and store
1575  * them in the @p flow if everything is correct.
1576  *
1577  * @param[in] dev
1578  *   Pointer to dev struct.
1579  * @param[in] attributes
1580  *   Pointer to flow attributes
1581  * @param[out] error
1582  *   Pointer to error structure.
1583  *
1584  * @return
1585  *   0 on success, a negative errno value otherwise and rte_errno is set.
1586  */
1587 static int
1588 flow_dv_validate_attributes(struct rte_eth_dev *dev,
1589                             const struct rte_flow_attr *attributes,
1590                             struct rte_flow_error *error)
1591 {
1592         struct priv *priv = dev->data->dev_private;
1593         uint32_t priority_max = priv->config.flow_prio - 1;
1594
1595         if (attributes->group)
1596                 return rte_flow_error_set(error, ENOTSUP,
1597                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1598                                           NULL,
1599                                           "groups is not supported");
1600         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1601             attributes->priority >= priority_max)
1602                 return rte_flow_error_set(error, ENOTSUP,
1603                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1604                                           NULL,
1605                                           "priority out of range");
1606         if (attributes->transfer)
1607                 return rte_flow_error_set(error, ENOTSUP,
1608                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1609                                           NULL,
1610                                           "transfer is not supported");
1611         if (!(attributes->egress ^ attributes->ingress))
1612                 return rte_flow_error_set(error, ENOTSUP,
1613                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
1614                                           "must specify exactly one of "
1615                                           "ingress or egress");
1616         return 0;
1617 }
1618
1619 /**
1620  * Internal validation function. For validating both actions and items.
1621  *
1622  * @param[in] dev
1623  *   Pointer to the rte_eth_dev structure.
1624  * @param[in] attr
1625  *   Pointer to the flow attributes.
1626  * @param[in] items
1627  *   Pointer to the list of items.
1628  * @param[in] actions
1629  *   Pointer to the list of actions.
1630  * @param[out] error
1631  *   Pointer to the error structure.
1632  *
1633  * @return
1634  *   0 on success, a negative errno value otherwise and rte_ernno is set.
1635  */
1636 static int
1637 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
1638                  const struct rte_flow_item items[],
1639                  const struct rte_flow_action actions[],
1640                  struct rte_flow_error *error)
1641 {
1642         int ret;
1643         uint64_t action_flags = 0;
1644         uint64_t item_flags = 0;
1645         uint64_t last_item = 0;
1646         int tunnel = 0;
1647         uint8_t next_protocol = 0xff;
1648         int actions_n = 0;
1649
1650         if (items == NULL)
1651                 return -1;
1652         ret = flow_dv_validate_attributes(dev, attr, error);
1653         if (ret < 0)
1654                 return ret;
1655         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1656                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1657                 switch (items->type) {
1658                 case RTE_FLOW_ITEM_TYPE_VOID:
1659                         break;
1660                 case RTE_FLOW_ITEM_TYPE_ETH:
1661                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1662                                                           error);
1663                         if (ret < 0)
1664                                 return ret;
1665                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1666                                              MLX5_FLOW_LAYER_OUTER_L2;
1667                         break;
1668                 case RTE_FLOW_ITEM_TYPE_VLAN:
1669                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1670                                                            error);
1671                         if (ret < 0)
1672                                 return ret;
1673                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1674                                              MLX5_FLOW_LAYER_OUTER_VLAN;
1675                         break;
1676                 case RTE_FLOW_ITEM_TYPE_IPV4:
1677                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1678                                                            error);
1679                         if (ret < 0)
1680                                 return ret;
1681                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1682                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1683                         if (items->mask != NULL &&
1684                             ((const struct rte_flow_item_ipv4 *)
1685                              items->mask)->hdr.next_proto_id) {
1686                                 next_protocol =
1687                                         ((const struct rte_flow_item_ipv4 *)
1688                                          (items->spec))->hdr.next_proto_id;
1689                                 next_protocol &=
1690                                         ((const struct rte_flow_item_ipv4 *)
1691                                          (items->mask))->hdr.next_proto_id;
1692                         } else {
1693                                 /* Reset for inner layer. */
1694                                 next_protocol = 0xff;
1695                         }
1696                         break;
1697                 case RTE_FLOW_ITEM_TYPE_IPV6:
1698                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1699                                                            error);
1700                         if (ret < 0)
1701                                 return ret;
1702                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1703                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1704                         if (items->mask != NULL &&
1705                             ((const struct rte_flow_item_ipv6 *)
1706                              items->mask)->hdr.proto) {
1707                                 next_protocol =
1708                                         ((const struct rte_flow_item_ipv6 *)
1709                                          items->spec)->hdr.proto;
1710                                 next_protocol &=
1711                                         ((const struct rte_flow_item_ipv6 *)
1712                                          items->mask)->hdr.proto;
1713                         } else {
1714                                 /* Reset for inner layer. */
1715                                 next_protocol = 0xff;
1716                         }
1717                         break;
1718                 case RTE_FLOW_ITEM_TYPE_TCP:
1719                         ret = mlx5_flow_validate_item_tcp
1720                                                 (items, item_flags,
1721                                                  next_protocol,
1722                                                  &rte_flow_item_tcp_mask,
1723                                                  error);
1724                         if (ret < 0)
1725                                 return ret;
1726                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1727                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1728                         break;
1729                 case RTE_FLOW_ITEM_TYPE_UDP:
1730                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1731                                                           next_protocol,
1732                                                           error);
1733                         if (ret < 0)
1734                                 return ret;
1735                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1736                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1737                         break;
1738                 case RTE_FLOW_ITEM_TYPE_GRE:
1739                 case RTE_FLOW_ITEM_TYPE_NVGRE:
1740                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1741                                                           next_protocol, error);
1742                         if (ret < 0)
1743                                 return ret;
1744                         last_item = MLX5_FLOW_LAYER_GRE;
1745                         break;
1746                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1747                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1748                                                             error);
1749                         if (ret < 0)
1750                                 return ret;
1751                         last_item = MLX5_FLOW_LAYER_VXLAN;
1752                         break;
1753                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1754                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1755                                                                 item_flags, dev,
1756                                                                 error);
1757                         if (ret < 0)
1758                                 return ret;
1759                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1760                         break;
1761                 case RTE_FLOW_ITEM_TYPE_MPLS:
1762                         ret = mlx5_flow_validate_item_mpls(dev, items,
1763                                                            item_flags,
1764                                                            last_item, error);
1765                         if (ret < 0)
1766                                 return ret;
1767                         last_item = MLX5_FLOW_LAYER_MPLS;
1768                         break;
1769                 case RTE_FLOW_ITEM_TYPE_META:
1770                         ret = flow_dv_validate_item_meta(dev, items, attr,
1771                                                          error);
1772                         if (ret < 0)
1773                                 return ret;
1774                         last_item = MLX5_FLOW_ITEM_METADATA;
1775                         break;
1776                 default:
1777                         return rte_flow_error_set(error, ENOTSUP,
1778                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1779                                                   NULL, "item not supported");
1780                 }
1781                 item_flags |= last_item;
1782         }
1783         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1784                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
1785                         return rte_flow_error_set(error, ENOTSUP,
1786                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1787                                                   actions, "too many actions");
1788                 switch (actions->type) {
1789                 case RTE_FLOW_ACTION_TYPE_VOID:
1790                         break;
1791                 case RTE_FLOW_ACTION_TYPE_FLAG:
1792                         ret = mlx5_flow_validate_action_flag(action_flags,
1793                                                              attr, error);
1794                         if (ret < 0)
1795                                 return ret;
1796                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1797                         ++actions_n;
1798                         break;
1799                 case RTE_FLOW_ACTION_TYPE_MARK:
1800                         ret = mlx5_flow_validate_action_mark(actions,
1801                                                              action_flags,
1802                                                              attr, error);
1803                         if (ret < 0)
1804                                 return ret;
1805                         action_flags |= MLX5_FLOW_ACTION_MARK;
1806                         ++actions_n;
1807                         break;
1808                 case RTE_FLOW_ACTION_TYPE_DROP:
1809                         ret = mlx5_flow_validate_action_drop(action_flags,
1810                                                              attr, error);
1811                         if (ret < 0)
1812                                 return ret;
1813                         action_flags |= MLX5_FLOW_ACTION_DROP;
1814                         ++actions_n;
1815                         break;
1816                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1817                         ret = mlx5_flow_validate_action_queue(actions,
1818                                                               action_flags, dev,
1819                                                               attr, error);
1820                         if (ret < 0)
1821                                 return ret;
1822                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1823                         ++actions_n;
1824                         break;
1825                 case RTE_FLOW_ACTION_TYPE_RSS:
1826                         ret = mlx5_flow_validate_action_rss(actions,
1827                                                             action_flags, dev,
1828                                                             attr, error);
1829                         if (ret < 0)
1830                                 return ret;
1831                         action_flags |= MLX5_FLOW_ACTION_RSS;
1832                         ++actions_n;
1833                         break;
1834                 case RTE_FLOW_ACTION_TYPE_COUNT:
1835                         ret = flow_dv_validate_action_count(dev, error);
1836                         if (ret < 0)
1837                                 return ret;
1838                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1839                         ++actions_n;
1840                         break;
1841                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1842                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
1843                         ret = flow_dv_validate_action_l2_encap(action_flags,
1844                                                                actions, attr,
1845                                                                error);
1846                         if (ret < 0)
1847                                 return ret;
1848                         action_flags |= actions->type ==
1849                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
1850                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
1851                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
1852                         ++actions_n;
1853                         break;
1854                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1855                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
1856                         ret = flow_dv_validate_action_l2_decap(action_flags,
1857                                                                attr, error);
1858                         if (ret < 0)
1859                                 return ret;
1860                         action_flags |= actions->type ==
1861                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
1862                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
1863                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
1864                         ++actions_n;
1865                         break;
1866                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
1867                         ret = flow_dv_validate_action_raw_encap(action_flags,
1868                                                                 actions, attr,
1869                                                                 error);
1870                         if (ret < 0)
1871                                 return ret;
1872                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
1873                         ++actions_n;
1874                         break;
1875                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
1876                         ret = flow_dv_validate_action_raw_decap(action_flags,
1877                                                                 actions, attr,
1878                                                                 error);
1879                         if (ret < 0)
1880                                 return ret;
1881                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
1882                         ++actions_n;
1883                         break;
1884                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
1885                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
1886                         ret = flow_dv_validate_action_modify_mac(action_flags,
1887                                                                  actions,
1888                                                                  item_flags,
1889                                                                  error);
1890                         if (ret < 0)
1891                                 return ret;
1892                         /* Count all modify-header actions as one action. */
1893                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1894                                 ++actions_n;
1895                         action_flags |= actions->type ==
1896                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
1897                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
1898                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
1899                         break;
1900
1901                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
1902                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
1903                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
1904                                                                   actions,
1905                                                                   item_flags,
1906                                                                   error);
1907                         if (ret < 0)
1908                                 return ret;
1909                         /* Count all modify-header actions as one action. */
1910                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1911                                 ++actions_n;
1912                         action_flags |= actions->type ==
1913                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
1914                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
1915                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
1916                         break;
1917                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
1918                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
1919                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
1920                                                                   actions,
1921                                                                   item_flags,
1922                                                                   error);
1923                         if (ret < 0)
1924                                 return ret;
1925                         /* Count all modify-header actions as one action. */
1926                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1927                                 ++actions_n;
1928                         action_flags |= actions->type ==
1929                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
1930                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
1931                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
1932                         break;
1933                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
1934                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
1935                         ret = flow_dv_validate_action_modify_tp(action_flags,
1936                                                                 actions,
1937                                                                 item_flags,
1938                                                                 error);
1939                         if (ret < 0)
1940                                 return ret;
1941                         /* Count all modify-header actions as one action. */
1942                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1943                                 ++actions_n;
1944                         action_flags |= actions->type ==
1945                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
1946                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
1947                                                 MLX5_FLOW_ACTION_SET_TP_DST;
1948                         break;
1949                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
1950                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
1951                         ret = flow_dv_validate_action_modify_ttl(action_flags,
1952                                                                  actions,
1953                                                                  item_flags,
1954                                                                  error);
1955                         if (ret < 0)
1956                                 return ret;
1957                         /* Count all modify-header actions as one action. */
1958                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
1959                                 ++actions_n;
1960                         action_flags |= actions->type ==
1961                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
1962                                                 MLX5_FLOW_ACTION_SET_TTL :
1963                                                 MLX5_FLOW_ACTION_DEC_TTL;
1964                         break;
1965                 default:
1966                         return rte_flow_error_set(error, ENOTSUP,
1967                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1968                                                   actions,
1969                                                   "action not supported");
1970                 }
1971         }
1972         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
1973                 return rte_flow_error_set(error, EINVAL,
1974                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1975                                           "no fate action is found");
1976         return 0;
1977 }
1978
1979 /**
1980  * Internal preparation function. Allocates the DV flow size,
1981  * this size is constant.
1982  *
1983  * @param[in] attr
1984  *   Pointer to the flow attributes.
1985  * @param[in] items
1986  *   Pointer to the list of items.
1987  * @param[in] actions
1988  *   Pointer to the list of actions.
1989  * @param[out] error
1990  *   Pointer to the error structure.
1991  *
1992  * @return
1993  *   Pointer to mlx5_flow object on success,
1994  *   otherwise NULL and rte_ernno is set.
1995  */
1996 static struct mlx5_flow *
1997 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
1998                 const struct rte_flow_item items[] __rte_unused,
1999                 const struct rte_flow_action actions[] __rte_unused,
2000                 struct rte_flow_error *error)
2001 {
2002         uint32_t size = sizeof(struct mlx5_flow);
2003         struct mlx5_flow *flow;
2004
2005         flow = rte_calloc(__func__, 1, size, 0);
2006         if (!flow) {
2007                 rte_flow_error_set(error, ENOMEM,
2008                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2009                                    "not enough memory to create flow");
2010                 return NULL;
2011         }
2012         flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
2013         return flow;
2014 }
2015
2016 #ifndef NDEBUG
2017 /**
2018  * Sanity check for match mask and value. Similar to check_valid_spec() in
2019  * kernel driver. If unmasked bit is present in value, it returns failure.
2020  *
2021  * @param match_mask
2022  *   pointer to match mask buffer.
2023  * @param match_value
2024  *   pointer to match value buffer.
2025  *
2026  * @return
2027  *   0 if valid, -EINVAL otherwise.
2028  */
2029 static int
2030 flow_dv_check_valid_spec(void *match_mask, void *match_value)
2031 {
2032         uint8_t *m = match_mask;
2033         uint8_t *v = match_value;
2034         unsigned int i;
2035
2036         for (i = 0; i < MLX5_ST_SZ_DB(fte_match_param); ++i) {
2037                 if (v[i] & ~m[i]) {
2038                         DRV_LOG(ERR,
2039                                 "match_value differs from match_criteria"
2040                                 " %p[%u] != %p[%u]",
2041                                 match_value, i, match_mask, i);
2042                         return -EINVAL;
2043                 }
2044         }
2045         return 0;
2046 }
2047 #endif
2048
2049 /**
2050  * Add Ethernet item to matcher and to the value.
2051  *
2052  * @param[in, out] matcher
2053  *   Flow matcher.
2054  * @param[in, out] key
2055  *   Flow matcher value.
2056  * @param[in] item
2057  *   Flow pattern to translate.
2058  * @param[in] inner
2059  *   Item is inner pattern.
2060  */
2061 static void
2062 flow_dv_translate_item_eth(void *matcher, void *key,
2063                            const struct rte_flow_item *item, int inner)
2064 {
2065         const struct rte_flow_item_eth *eth_m = item->mask;
2066         const struct rte_flow_item_eth *eth_v = item->spec;
2067         const struct rte_flow_item_eth nic_mask = {
2068                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2069                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2070                 .type = RTE_BE16(0xffff),
2071         };
2072         void *headers_m;
2073         void *headers_v;
2074         char *l24_v;
2075         unsigned int i;
2076
2077         if (!eth_v)
2078                 return;
2079         if (!eth_m)
2080                 eth_m = &nic_mask;
2081         if (inner) {
2082                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2083                                          inner_headers);
2084                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2085         } else {
2086                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2087                                          outer_headers);
2088                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2089         }
2090         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
2091                &eth_m->dst, sizeof(eth_m->dst));
2092         /* The value must be in the range of the mask. */
2093         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
2094         for (i = 0; i < sizeof(eth_m->dst); ++i)
2095                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
2096         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
2097                &eth_m->src, sizeof(eth_m->src));
2098         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
2099         /* The value must be in the range of the mask. */
2100         for (i = 0; i < sizeof(eth_m->dst); ++i)
2101                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
2102         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
2103                  rte_be_to_cpu_16(eth_m->type));
2104         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
2105         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
2106 }
2107
2108 /**
2109  * Add VLAN item to matcher and to the value.
2110  *
2111  * @param[in, out] matcher
2112  *   Flow matcher.
2113  * @param[in, out] key
2114  *   Flow matcher value.
2115  * @param[in] item
2116  *   Flow pattern to translate.
2117  * @param[in] inner
2118  *   Item is inner pattern.
2119  */
2120 static void
2121 flow_dv_translate_item_vlan(void *matcher, void *key,
2122                             const struct rte_flow_item *item,
2123                             int inner)
2124 {
2125         const struct rte_flow_item_vlan *vlan_m = item->mask;
2126         const struct rte_flow_item_vlan *vlan_v = item->spec;
2127         const struct rte_flow_item_vlan nic_mask = {
2128                 .tci = RTE_BE16(0x0fff),
2129                 .inner_type = RTE_BE16(0xffff),
2130         };
2131         void *headers_m;
2132         void *headers_v;
2133         uint16_t tci_m;
2134         uint16_t tci_v;
2135
2136         if (!vlan_v)
2137                 return;
2138         if (!vlan_m)
2139                 vlan_m = &nic_mask;
2140         if (inner) {
2141                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2142                                          inner_headers);
2143                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2144         } else {
2145                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2146                                          outer_headers);
2147                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2148         }
2149         tci_m = rte_be_to_cpu_16(vlan_m->tci);
2150         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
2151         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
2152         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
2153         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
2154         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
2155         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
2156         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
2157         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
2158         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
2159 }
2160
2161 /**
2162  * Add IPV4 item to matcher and to the value.
2163  *
2164  * @param[in, out] matcher
2165  *   Flow matcher.
2166  * @param[in, out] key
2167  *   Flow matcher value.
2168  * @param[in] item
2169  *   Flow pattern to translate.
2170  * @param[in] inner
2171  *   Item is inner pattern.
2172  */
2173 static void
2174 flow_dv_translate_item_ipv4(void *matcher, void *key,
2175                             const struct rte_flow_item *item,
2176                             int inner)
2177 {
2178         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
2179         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
2180         const struct rte_flow_item_ipv4 nic_mask = {
2181                 .hdr = {
2182                         .src_addr = RTE_BE32(0xffffffff),
2183                         .dst_addr = RTE_BE32(0xffffffff),
2184                         .type_of_service = 0xff,
2185                         .next_proto_id = 0xff,
2186                 },
2187         };
2188         void *headers_m;
2189         void *headers_v;
2190         char *l24_m;
2191         char *l24_v;
2192         uint8_t tos;
2193
2194         if (inner) {
2195                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2196                                          inner_headers);
2197                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2198         } else {
2199                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2200                                          outer_headers);
2201                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2202         }
2203         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
2204         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
2205         if (!ipv4_v)
2206                 return;
2207         if (!ipv4_m)
2208                 ipv4_m = &nic_mask;
2209         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2210                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2211         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2212                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2213         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
2214         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
2215         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2216                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
2217         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2218                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
2219         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
2220         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
2221         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
2222         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
2223                  ipv4_m->hdr.type_of_service);
2224         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
2225         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
2226                  ipv4_m->hdr.type_of_service >> 2);
2227         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
2228         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
2229                  ipv4_m->hdr.next_proto_id);
2230         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2231                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
2232 }
2233
2234 /**
2235  * Add IPV6 item to matcher and to the value.
2236  *
2237  * @param[in, out] matcher
2238  *   Flow matcher.
2239  * @param[in, out] key
2240  *   Flow matcher value.
2241  * @param[in] item
2242  *   Flow pattern to translate.
2243  * @param[in] inner
2244  *   Item is inner pattern.
2245  */
2246 static void
2247 flow_dv_translate_item_ipv6(void *matcher, void *key,
2248                             const struct rte_flow_item *item,
2249                             int inner)
2250 {
2251         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
2252         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
2253         const struct rte_flow_item_ipv6 nic_mask = {
2254                 .hdr = {
2255                         .src_addr =
2256                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2257                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2258                         .dst_addr =
2259                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2260                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2261                         .vtc_flow = RTE_BE32(0xffffffff),
2262                         .proto = 0xff,
2263                         .hop_limits = 0xff,
2264                 },
2265         };
2266         void *headers_m;
2267         void *headers_v;
2268         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2269         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2270         char *l24_m;
2271         char *l24_v;
2272         uint32_t vtc_m;
2273         uint32_t vtc_v;
2274         int i;
2275         int size;
2276
2277         if (inner) {
2278                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2279                                          inner_headers);
2280                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2281         } else {
2282                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2283                                          outer_headers);
2284                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2285         }
2286         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
2287         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
2288         if (!ipv6_v)
2289                 return;
2290         if (!ipv6_m)
2291                 ipv6_m = &nic_mask;
2292         size = sizeof(ipv6_m->hdr.dst_addr);
2293         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2294                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
2295         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2296                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
2297         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
2298         for (i = 0; i < size; ++i)
2299                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
2300         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2301                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
2302         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2303                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
2304         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
2305         for (i = 0; i < size; ++i)
2306                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
2307         /* TOS. */
2308         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
2309         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
2310         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
2311         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
2312         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
2313         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
2314         /* Label. */
2315         if (inner) {
2316                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
2317                          vtc_m);
2318                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
2319                          vtc_v);
2320         } else {
2321                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
2322                          vtc_m);
2323                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
2324                          vtc_v);
2325         }
2326         /* Protocol. */
2327         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
2328                  ipv6_m->hdr.proto);
2329         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2330                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
2331 }
2332
2333 /**
2334  * Add TCP item to matcher and to the value.
2335  *
2336  * @param[in, out] matcher
2337  *   Flow matcher.
2338  * @param[in, out] key
2339  *   Flow matcher value.
2340  * @param[in] item
2341  *   Flow pattern to translate.
2342  * @param[in] inner
2343  *   Item is inner pattern.
2344  */
2345 static void
2346 flow_dv_translate_item_tcp(void *matcher, void *key,
2347                            const struct rte_flow_item *item,
2348                            int inner)
2349 {
2350         const struct rte_flow_item_tcp *tcp_m = item->mask;
2351         const struct rte_flow_item_tcp *tcp_v = item->spec;
2352         void *headers_m;
2353         void *headers_v;
2354
2355         if (inner) {
2356                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2357                                          inner_headers);
2358                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2359         } else {
2360                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2361                                          outer_headers);
2362                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2363         }
2364         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2365         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
2366         if (!tcp_v)
2367                 return;
2368         if (!tcp_m)
2369                 tcp_m = &rte_flow_item_tcp_mask;
2370         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
2371                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
2372         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
2373                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
2374         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
2375                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
2376         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
2377                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
2378 }
2379
2380 /**
2381  * Add UDP item to matcher and to the value.
2382  *
2383  * @param[in, out] matcher
2384  *   Flow matcher.
2385  * @param[in, out] key
2386  *   Flow matcher value.
2387  * @param[in] item
2388  *   Flow pattern to translate.
2389  * @param[in] inner
2390  *   Item is inner pattern.
2391  */
2392 static void
2393 flow_dv_translate_item_udp(void *matcher, void *key,
2394                            const struct rte_flow_item *item,
2395                            int inner)
2396 {
2397         const struct rte_flow_item_udp *udp_m = item->mask;
2398         const struct rte_flow_item_udp *udp_v = item->spec;
2399         void *headers_m;
2400         void *headers_v;
2401
2402         if (inner) {
2403                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2404                                          inner_headers);
2405                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2406         } else {
2407                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2408                                          outer_headers);
2409                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2410         }
2411         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2412         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
2413         if (!udp_v)
2414                 return;
2415         if (!udp_m)
2416                 udp_m = &rte_flow_item_udp_mask;
2417         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
2418                  rte_be_to_cpu_16(udp_m->hdr.src_port));
2419         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
2420                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
2421         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
2422                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
2423         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2424                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
2425 }
2426
2427 /**
2428  * Add GRE item to matcher and to the value.
2429  *
2430  * @param[in, out] matcher
2431  *   Flow matcher.
2432  * @param[in, out] key
2433  *   Flow matcher value.
2434  * @param[in] item
2435  *   Flow pattern to translate.
2436  * @param[in] inner
2437  *   Item is inner pattern.
2438  */
2439 static void
2440 flow_dv_translate_item_gre(void *matcher, void *key,
2441                            const struct rte_flow_item *item,
2442                            int inner)
2443 {
2444         const struct rte_flow_item_gre *gre_m = item->mask;
2445         const struct rte_flow_item_gre *gre_v = item->spec;
2446         void *headers_m;
2447         void *headers_v;
2448         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2449         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2450
2451         if (inner) {
2452                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2453                                          inner_headers);
2454                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2455         } else {
2456                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2457                                          outer_headers);
2458                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2459         }
2460         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2461         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
2462         if (!gre_v)
2463                 return;
2464         if (!gre_m)
2465                 gre_m = &rte_flow_item_gre_mask;
2466         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
2467                  rte_be_to_cpu_16(gre_m->protocol));
2468         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
2469                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
2470 }
2471
2472 /**
2473  * Add NVGRE item to matcher and to the value.
2474  *
2475  * @param[in, out] matcher
2476  *   Flow matcher.
2477  * @param[in, out] key
2478  *   Flow matcher value.
2479  * @param[in] item
2480  *   Flow pattern to translate.
2481  * @param[in] inner
2482  *   Item is inner pattern.
2483  */
2484 static void
2485 flow_dv_translate_item_nvgre(void *matcher, void *key,
2486                              const struct rte_flow_item *item,
2487                              int inner)
2488 {
2489         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
2490         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
2491         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2492         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2493         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
2494         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
2495         char *gre_key_m;
2496         char *gre_key_v;
2497         int size;
2498         int i;
2499
2500         flow_dv_translate_item_gre(matcher, key, item, inner);
2501         if (!nvgre_v)
2502                 return;
2503         if (!nvgre_m)
2504                 nvgre_m = &rte_flow_item_nvgre_mask;
2505         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
2506         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
2507         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
2508         memcpy(gre_key_m, tni_flow_id_m, size);
2509         for (i = 0; i < size; ++i)
2510                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
2511 }
2512
2513 /**
2514  * Add VXLAN item to matcher and to the value.
2515  *
2516  * @param[in, out] matcher
2517  *   Flow matcher.
2518  * @param[in, out] key
2519  *   Flow matcher value.
2520  * @param[in] item
2521  *   Flow pattern to translate.
2522  * @param[in] inner
2523  *   Item is inner pattern.
2524  */
2525 static void
2526 flow_dv_translate_item_vxlan(void *matcher, void *key,
2527                              const struct rte_flow_item *item,
2528                              int inner)
2529 {
2530         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
2531         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
2532         void *headers_m;
2533         void *headers_v;
2534         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2535         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2536         char *vni_m;
2537         char *vni_v;
2538         uint16_t dport;
2539         int size;
2540         int i;
2541
2542         if (inner) {
2543                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2544                                          inner_headers);
2545                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2546         } else {
2547                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2548                                          outer_headers);
2549                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2550         }
2551         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
2552                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
2553         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
2554                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
2555                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
2556         }
2557         if (!vxlan_v)
2558                 return;
2559         if (!vxlan_m)
2560                 vxlan_m = &rte_flow_item_vxlan_mask;
2561         size = sizeof(vxlan_m->vni);
2562         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
2563         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
2564         memcpy(vni_m, vxlan_m->vni, size);
2565         for (i = 0; i < size; ++i)
2566                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
2567 }
2568
2569 /**
2570  * Add MPLS item to matcher and to the value.
2571  *
2572  * @param[in, out] matcher
2573  *   Flow matcher.
2574  * @param[in, out] key
2575  *   Flow matcher value.
2576  * @param[in] item
2577  *   Flow pattern to translate.
2578  * @param[in] prev_layer
2579  *   The protocol layer indicated in previous item.
2580  * @param[in] inner
2581  *   Item is inner pattern.
2582  */
2583 static void
2584 flow_dv_translate_item_mpls(void *matcher, void *key,
2585                             const struct rte_flow_item *item,
2586                             uint64_t prev_layer,
2587                             int inner)
2588 {
2589         const uint32_t *in_mpls_m = item->mask;
2590         const uint32_t *in_mpls_v = item->spec;
2591         uint32_t *out_mpls_m = 0;
2592         uint32_t *out_mpls_v = 0;
2593         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
2594         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
2595         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
2596                                      misc_parameters_2);
2597         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
2598         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
2599         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2600
2601         switch (prev_layer) {
2602         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
2603                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
2604                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2605                          MLX5_UDP_PORT_MPLS);
2606                 break;
2607         case MLX5_FLOW_LAYER_GRE:
2608                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
2609                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
2610                          ETHER_TYPE_MPLS);
2611                 break;
2612         default:
2613                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
2614                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2615                          IPPROTO_MPLS);
2616                 break;
2617         }
2618         if (!in_mpls_v)
2619                 return;
2620         if (!in_mpls_m)
2621                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
2622         switch (prev_layer) {
2623         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
2624                 out_mpls_m =
2625                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
2626                                                  outer_first_mpls_over_udp);
2627                 out_mpls_v =
2628                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
2629                                                  outer_first_mpls_over_udp);
2630                 break;
2631         case MLX5_FLOW_LAYER_GRE:
2632                 out_mpls_m =
2633                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
2634                                                  outer_first_mpls_over_gre);
2635                 out_mpls_v =
2636                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
2637                                                  outer_first_mpls_over_gre);
2638                 break;
2639         default:
2640                 /* Inner MPLS not over GRE is not supported. */
2641                 if (!inner) {
2642                         out_mpls_m =
2643                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
2644                                                          misc2_m,
2645                                                          outer_first_mpls);
2646                         out_mpls_v =
2647                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
2648                                                          misc2_v,
2649                                                          outer_first_mpls);
2650                 }
2651                 break;
2652         }
2653         if (out_mpls_m && out_mpls_v) {
2654                 *out_mpls_m = *in_mpls_m;
2655                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
2656         }
2657 }
2658
2659 /**
2660  * Add META item to matcher
2661  *
2662  * @param[in, out] matcher
2663  *   Flow matcher.
2664  * @param[in, out] key
2665  *   Flow matcher value.
2666  * @param[in] item
2667  *   Flow pattern to translate.
2668  * @param[in] inner
2669  *   Item is inner pattern.
2670  */
2671 static void
2672 flow_dv_translate_item_meta(void *matcher, void *key,
2673                             const struct rte_flow_item *item)
2674 {
2675         const struct rte_flow_item_meta *meta_m;
2676         const struct rte_flow_item_meta *meta_v;
2677         void *misc2_m =
2678                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
2679         void *misc2_v =
2680                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
2681
2682         meta_m = (const void *)item->mask;
2683         if (!meta_m)
2684                 meta_m = &rte_flow_item_meta_mask;
2685         meta_v = (const void *)item->spec;
2686         if (meta_v) {
2687                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
2688                          rte_be_to_cpu_32(meta_m->data));
2689                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
2690                          rte_be_to_cpu_32(meta_v->data & meta_m->data));
2691         }
2692 }
2693
2694 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
2695
2696 #define HEADER_IS_ZERO(match_criteria, headers)                              \
2697         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
2698                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
2699
2700 /**
2701  * Calculate flow matcher enable bitmap.
2702  *
2703  * @param match_criteria
2704  *   Pointer to flow matcher criteria.
2705  *
2706  * @return
2707  *   Bitmap of enabled fields.
2708  */
2709 static uint8_t
2710 flow_dv_matcher_enable(uint32_t *match_criteria)
2711 {
2712         uint8_t match_criteria_enable;
2713
2714         match_criteria_enable =
2715                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
2716                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
2717         match_criteria_enable |=
2718                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
2719                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
2720         match_criteria_enable |=
2721                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
2722                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
2723         match_criteria_enable |=
2724                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
2725                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
2726
2727         return match_criteria_enable;
2728 }
2729
2730 /**
2731  * Register the flow matcher.
2732  *
2733  * @param dev[in, out]
2734  *   Pointer to rte_eth_dev structure.
2735  * @param[in, out] matcher
2736  *   Pointer to flow matcher.
2737  * @parm[in, out] dev_flow
2738  *   Pointer to the dev_flow.
2739  * @param[out] error
2740  *   pointer to error structure.
2741  *
2742  * @return
2743  *   0 on success otherwise -errno and errno is set.
2744  */
2745 static int
2746 flow_dv_matcher_register(struct rte_eth_dev *dev,
2747                          struct mlx5_flow_dv_matcher *matcher,
2748                          struct mlx5_flow *dev_flow,
2749                          struct rte_flow_error *error)
2750 {
2751         struct priv *priv = dev->data->dev_private;
2752         struct mlx5_flow_dv_matcher *cache_matcher;
2753         struct mlx5dv_flow_matcher_attr dv_attr = {
2754                 .type = IBV_FLOW_ATTR_NORMAL,
2755                 .match_mask = (void *)&matcher->mask,
2756         };
2757
2758         /* Lookup from cache. */
2759         LIST_FOREACH(cache_matcher, &priv->matchers, next) {
2760                 if (matcher->crc == cache_matcher->crc &&
2761                     matcher->priority == cache_matcher->priority &&
2762                     matcher->egress == cache_matcher->egress &&
2763                     !memcmp((const void *)matcher->mask.buf,
2764                             (const void *)cache_matcher->mask.buf,
2765                             cache_matcher->mask.size)) {
2766                         DRV_LOG(DEBUG,
2767                                 "priority %hd use %s matcher %p: refcnt %d++",
2768                                 cache_matcher->priority,
2769                                 cache_matcher->egress ? "tx" : "rx",
2770                                 (void *)cache_matcher,
2771                                 rte_atomic32_read(&cache_matcher->refcnt));
2772                         rte_atomic32_inc(&cache_matcher->refcnt);
2773                         dev_flow->dv.matcher = cache_matcher;
2774                         return 0;
2775                 }
2776         }
2777         /* Register new matcher. */
2778         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
2779         if (!cache_matcher)
2780                 return rte_flow_error_set(error, ENOMEM,
2781                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2782                                           "cannot allocate matcher memory");
2783         *cache_matcher = *matcher;
2784         dv_attr.match_criteria_enable =
2785                 flow_dv_matcher_enable(cache_matcher->mask.buf);
2786         dv_attr.priority = matcher->priority;
2787         if (matcher->egress)
2788                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
2789         cache_matcher->matcher_object =
2790                 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
2791         if (!cache_matcher->matcher_object) {
2792                 rte_free(cache_matcher);
2793                 return rte_flow_error_set(error, ENOMEM,
2794                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2795                                           NULL, "cannot create matcher");
2796         }
2797         rte_atomic32_inc(&cache_matcher->refcnt);
2798         LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
2799         dev_flow->dv.matcher = cache_matcher;
2800         DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
2801                 cache_matcher->priority,
2802                 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
2803                 rte_atomic32_read(&cache_matcher->refcnt));
2804         return 0;
2805 }
2806
2807 /**
2808  * Fill the flow with DV spec.
2809  *
2810  * @param[in] dev
2811  *   Pointer to rte_eth_dev structure.
2812  * @param[in, out] dev_flow
2813  *   Pointer to the sub flow.
2814  * @param[in] attr
2815  *   Pointer to the flow attributes.
2816  * @param[in] items
2817  *   Pointer to the list of items.
2818  * @param[in] actions
2819  *   Pointer to the list of actions.
2820  * @param[out] error
2821  *   Pointer to the error structure.
2822  *
2823  * @return
2824  *   0 on success, a negative errno value otherwise and rte_ernno is set.
2825  */
2826 static int
2827 flow_dv_translate(struct rte_eth_dev *dev,
2828                   struct mlx5_flow *dev_flow,
2829                   const struct rte_flow_attr *attr,
2830                   const struct rte_flow_item items[],
2831                   const struct rte_flow_action actions[],
2832                   struct rte_flow_error *error)
2833 {
2834         struct priv *priv = dev->data->dev_private;
2835         struct rte_flow *flow = dev_flow->flow;
2836         uint64_t item_flags = 0;
2837         uint64_t last_item = 0;
2838         uint64_t action_flags = 0;
2839         uint64_t priority = attr->priority;
2840         struct mlx5_flow_dv_matcher matcher = {
2841                 .mask = {
2842                         .size = sizeof(matcher.mask.buf),
2843                 },
2844         };
2845         int actions_n = 0;
2846         bool actions_end = false;
2847         struct mlx5_flow_dv_modify_hdr_resource res = {
2848                 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
2849                                           MLX5DV_FLOW_TABLE_TYPE_NIC_RX
2850         };
2851         union flow_dv_attr flow_attr = { .attr = 0 };
2852
2853         if (priority == MLX5_FLOW_PRIO_RSVD)
2854                 priority = priv->config.flow_prio - 1;
2855         for (; !actions_end ; actions++) {
2856                 const struct rte_flow_action_queue *queue;
2857                 const struct rte_flow_action_rss *rss;
2858                 const struct rte_flow_action *action = actions;
2859                 const struct rte_flow_action_count *count = action->conf;
2860                 const uint8_t *rss_key;
2861
2862                 switch (actions->type) {
2863                 case RTE_FLOW_ACTION_TYPE_VOID:
2864                         break;
2865                 case RTE_FLOW_ACTION_TYPE_FLAG:
2866                         dev_flow->dv.actions[actions_n].type =
2867                                 MLX5DV_FLOW_ACTION_TAG;
2868                         dev_flow->dv.actions[actions_n].tag_value =
2869                                 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
2870                         actions_n++;
2871                         action_flags |= MLX5_FLOW_ACTION_FLAG;
2872                         break;
2873                 case RTE_FLOW_ACTION_TYPE_MARK:
2874                         dev_flow->dv.actions[actions_n].type =
2875                                 MLX5DV_FLOW_ACTION_TAG;
2876                         dev_flow->dv.actions[actions_n].tag_value =
2877                                 mlx5_flow_mark_set
2878                                 (((const struct rte_flow_action_mark *)
2879                                   (actions->conf))->id);
2880                         actions_n++;
2881                         action_flags |= MLX5_FLOW_ACTION_MARK;
2882                         break;
2883                 case RTE_FLOW_ACTION_TYPE_DROP:
2884                         dev_flow->dv.actions[actions_n].type =
2885                                 MLX5DV_FLOW_ACTION_DROP;
2886                         action_flags |= MLX5_FLOW_ACTION_DROP;
2887                         break;
2888                 case RTE_FLOW_ACTION_TYPE_QUEUE:
2889                         queue = actions->conf;
2890                         flow->rss.queue_num = 1;
2891                         (*flow->queue)[0] = queue->index;
2892                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
2893                         break;
2894                 case RTE_FLOW_ACTION_TYPE_RSS:
2895                         rss = actions->conf;
2896                         if (flow->queue)
2897                                 memcpy((*flow->queue), rss->queue,
2898                                        rss->queue_num * sizeof(uint16_t));
2899                         flow->rss.queue_num = rss->queue_num;
2900                         /* NULL RSS key indicates default RSS key. */
2901                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
2902                         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
2903                         /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
2904                         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
2905                         flow->rss.level = rss->level;
2906                         action_flags |= MLX5_FLOW_ACTION_RSS;
2907                         break;
2908                 case RTE_FLOW_ACTION_TYPE_COUNT:
2909                         if (!priv->config.devx) {
2910                                 rte_errno = ENOTSUP;
2911                                 goto cnt_err;
2912                         }
2913                         flow->counter =
2914                                 flow_dv_counter_new(dev,
2915                                                     count->shared, count->id);
2916                         if (flow->counter == NULL)
2917                                 goto cnt_err;
2918                         dev_flow->dv.actions[actions_n].type =
2919                                         MLX5DV_FLOW_ACTION_COUNTER_DEVX;
2920                         dev_flow->dv.actions[actions_n].obj =
2921                                                 flow->counter->dcs->obj;
2922                         action_flags |= MLX5_FLOW_ACTION_COUNT;
2923                         ++actions_n;
2924                         break;
2925 cnt_err:
2926                         if (rte_errno == ENOTSUP)
2927                                 return rte_flow_error_set
2928                                               (error, ENOTSUP,
2929                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2930                                                NULL,
2931                                                "count action not supported");
2932                         else
2933                                 return rte_flow_error_set
2934                                                 (error, rte_errno,
2935                                                  RTE_FLOW_ERROR_TYPE_ACTION,
2936                                                  action,
2937                                                  "cannot create counter"
2938                                                   " object.");
2939                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2940                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2941                         if (flow_dv_create_action_l2_encap(dev, actions,
2942                                                            dev_flow, error))
2943                                 return -rte_errno;
2944                         dev_flow->dv.actions[actions_n].type =
2945                                 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
2946                         dev_flow->dv.actions[actions_n].action =
2947                                 dev_flow->dv.encap_decap->verbs_action;
2948                         actions_n++;
2949                         action_flags |= actions->type ==
2950                                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
2951                                         MLX5_FLOW_ACTION_VXLAN_ENCAP :
2952                                         MLX5_FLOW_ACTION_NVGRE_ENCAP;
2953                         break;
2954                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2955                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
2956                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
2957                                                            error))
2958                                 return -rte_errno;
2959                         dev_flow->dv.actions[actions_n].type =
2960                                 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
2961                         dev_flow->dv.actions[actions_n].action =
2962                                 dev_flow->dv.encap_decap->verbs_action;
2963                         actions_n++;
2964                         action_flags |= actions->type ==
2965                                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
2966                                         MLX5_FLOW_ACTION_VXLAN_DECAP :
2967                                         MLX5_FLOW_ACTION_NVGRE_DECAP;
2968                         break;
2969                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2970                         /* Handle encap with preceding decap. */
2971                         if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
2972                                 if (flow_dv_create_action_raw_encap
2973                                         (dev, actions, dev_flow, attr, error))
2974                                         return -rte_errno;
2975                                 dev_flow->dv.actions[actions_n].type =
2976                                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
2977                                 dev_flow->dv.actions[actions_n].action =
2978                                         dev_flow->dv.encap_decap->verbs_action;
2979                         } else {
2980                                 /* Handle encap without preceding decap. */
2981                                 if (flow_dv_create_action_l2_encap(dev, actions,
2982                                                                    dev_flow,
2983                                                                    error))
2984                                         return -rte_errno;
2985                                 dev_flow->dv.actions[actions_n].type =
2986                                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
2987                                 dev_flow->dv.actions[actions_n].action =
2988                                         dev_flow->dv.encap_decap->verbs_action;
2989                         }
2990                         actions_n++;
2991                         action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
2992                         break;
2993                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
2994                         /* Check if this decap is followed by encap. */
2995                         for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
2996                                action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
2997                                action++) {
2998                         }
2999                         /* Handle decap only if it isn't followed by encap. */
3000                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3001                                 if (flow_dv_create_action_l2_decap(dev,
3002                                                                    dev_flow,
3003                                                                    error))
3004                                         return -rte_errno;
3005                                 dev_flow->dv.actions[actions_n].type =
3006                                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
3007                                 dev_flow->dv.actions[actions_n].action =
3008                                         dev_flow->dv.encap_decap->verbs_action;
3009                                 actions_n++;
3010                         }
3011                         /* If decap is followed by encap, handle it at encap. */
3012                         action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3013                         break;
3014                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3015                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3016                         if (flow_dv_convert_action_modify_mac(&res, actions,
3017                                                               error))
3018                                 return -rte_errno;
3019                         action_flags |= actions->type ==
3020                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3021                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
3022                                         MLX5_FLOW_ACTION_SET_MAC_DST;
3023                         break;
3024                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3025                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3026                         if (flow_dv_convert_action_modify_ipv4(&res, actions,
3027                                                                error))
3028                                 return -rte_errno;
3029                         action_flags |= actions->type ==
3030                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3031                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
3032                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
3033                         break;
3034                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3035                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3036                         if (flow_dv_convert_action_modify_ipv6(&res, actions,
3037                                                                error))
3038                                 return -rte_errno;
3039                         action_flags |= actions->type ==
3040                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3041                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
3042                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
3043                         break;
3044                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3045                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3046                         if (flow_dv_convert_action_modify_tp(&res, actions,
3047                                                              items, &flow_attr,
3048                                                              error))
3049                                 return -rte_errno;
3050                         action_flags |= actions->type ==
3051                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3052                                         MLX5_FLOW_ACTION_SET_TP_SRC :
3053                                         MLX5_FLOW_ACTION_SET_TP_DST;
3054                         break;
3055                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3056                         if (flow_dv_convert_action_modify_dec_ttl(&res, items,
3057                                                                   &flow_attr,
3058                                                                   error))
3059                                 return -rte_errno;
3060                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
3061                         break;
3062                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3063                         if (flow_dv_convert_action_modify_ttl(&res, actions,
3064                                                              items, &flow_attr,
3065                                                              error))
3066                                 return -rte_errno;
3067                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
3068                         break;
3069                 case RTE_FLOW_ACTION_TYPE_END:
3070                         actions_end = true;
3071                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
3072                                 /* create modify action if needed. */
3073                                 if (flow_dv_modify_hdr_resource_register
3074                                                                 (dev, &res,
3075                                                                  dev_flow,
3076                                                                  error))
3077                                         return -rte_errno;
3078                                 dev_flow->dv.actions[actions_n].type =
3079                                         MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
3080                                 dev_flow->dv.actions[actions_n].action =
3081                                         dev_flow->dv.modify_hdr->verbs_action;
3082                                 actions_n++;
3083                         }
3084                         break;
3085                 default:
3086                         break;
3087                 }
3088         }
3089         dev_flow->dv.actions_n = actions_n;
3090         flow->actions = action_flags;
3091         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3092                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3093                 void *match_mask = matcher.mask.buf;
3094                 void *match_value = dev_flow->dv.value.buf;
3095
3096                 switch (items->type) {
3097                 case RTE_FLOW_ITEM_TYPE_ETH:
3098                         flow_dv_translate_item_eth(match_mask, match_value,
3099                                                    items, tunnel);
3100                         matcher.priority = MLX5_PRIORITY_MAP_L2;
3101                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
3102                                              MLX5_FLOW_LAYER_OUTER_L2;
3103                         break;
3104                 case RTE_FLOW_ITEM_TYPE_VLAN:
3105                         flow_dv_translate_item_vlan(match_mask, match_value,
3106                                                     items, tunnel);
3107                         matcher.priority = MLX5_PRIORITY_MAP_L2;
3108                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
3109                                               MLX5_FLOW_LAYER_INNER_VLAN) :
3110                                              (MLX5_FLOW_LAYER_OUTER_L2 |
3111                                               MLX5_FLOW_LAYER_OUTER_VLAN);
3112                         break;
3113                 case RTE_FLOW_ITEM_TYPE_IPV4:
3114                         flow_dv_translate_item_ipv4(match_mask, match_value,
3115                                                     items, tunnel);
3116                         matcher.priority = MLX5_PRIORITY_MAP_L3;
3117                         dev_flow->dv.hash_fields |=
3118                                 mlx5_flow_hashfields_adjust
3119                                         (dev_flow, tunnel,
3120                                          MLX5_IPV4_LAYER_TYPES,
3121                                          MLX5_IPV4_IBV_RX_HASH);
3122                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3123                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3124                         break;
3125                 case RTE_FLOW_ITEM_TYPE_IPV6:
3126                         flow_dv_translate_item_ipv6(match_mask, match_value,
3127                                                     items, tunnel);
3128                         matcher.priority = MLX5_PRIORITY_MAP_L3;
3129                         dev_flow->dv.hash_fields |=
3130                                 mlx5_flow_hashfields_adjust
3131                                         (dev_flow, tunnel,
3132                                          MLX5_IPV6_LAYER_TYPES,
3133                                          MLX5_IPV6_IBV_RX_HASH);
3134                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3135                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3136                         break;
3137                 case RTE_FLOW_ITEM_TYPE_TCP:
3138                         flow_dv_translate_item_tcp(match_mask, match_value,
3139                                                    items, tunnel);
3140                         matcher.priority = MLX5_PRIORITY_MAP_L4;
3141                         dev_flow->dv.hash_fields |=
3142                                 mlx5_flow_hashfields_adjust
3143                                         (dev_flow, tunnel, ETH_RSS_TCP,
3144                                          IBV_RX_HASH_SRC_PORT_TCP |
3145                                          IBV_RX_HASH_DST_PORT_TCP);
3146                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3147                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
3148                         break;
3149                 case RTE_FLOW_ITEM_TYPE_UDP:
3150                         flow_dv_translate_item_udp(match_mask, match_value,
3151                                                    items, tunnel);
3152                         matcher.priority = MLX5_PRIORITY_MAP_L4;
3153                         dev_flow->dv.hash_fields |=
3154                                 mlx5_flow_hashfields_adjust
3155                                         (dev_flow, tunnel, ETH_RSS_UDP,
3156                                          IBV_RX_HASH_SRC_PORT_UDP |
3157                                          IBV_RX_HASH_DST_PORT_UDP);
3158                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3159                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
3160                         break;
3161                 case RTE_FLOW_ITEM_TYPE_GRE:
3162                         flow_dv_translate_item_gre(match_mask, match_value,
3163                                                    items, tunnel);
3164                         last_item = MLX5_FLOW_LAYER_GRE;
3165                         break;
3166                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3167                         flow_dv_translate_item_nvgre(match_mask, match_value,
3168                                                      items, tunnel);
3169                         last_item = MLX5_FLOW_LAYER_GRE;
3170                         break;
3171                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3172                         flow_dv_translate_item_vxlan(match_mask, match_value,
3173                                                      items, tunnel);
3174                         last_item = MLX5_FLOW_LAYER_VXLAN;
3175                         break;
3176                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3177                         flow_dv_translate_item_vxlan(match_mask, match_value,
3178                                                      items, tunnel);
3179                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3180                         break;
3181                 case RTE_FLOW_ITEM_TYPE_MPLS:
3182                         flow_dv_translate_item_mpls(match_mask, match_value,
3183                                                     items, last_item, tunnel);
3184                         last_item = MLX5_FLOW_LAYER_MPLS;
3185                         break;
3186                 case RTE_FLOW_ITEM_TYPE_META:
3187                         flow_dv_translate_item_meta(match_mask, match_value,
3188                                                     items);
3189                         last_item = MLX5_FLOW_ITEM_METADATA;
3190                         break;
3191                 default:
3192                         break;
3193                 }
3194                 item_flags |= last_item;
3195         }
3196         assert(!flow_dv_check_valid_spec(matcher.mask.buf,
3197                                          dev_flow->dv.value.buf));
3198         dev_flow->layers = item_flags;
3199         /* Register matcher. */
3200         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
3201                                     matcher.mask.size);
3202         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
3203                                                      matcher.priority);
3204         matcher.egress = attr->egress;
3205         if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
3206                 return -rte_errno;
3207         return 0;
3208 }
3209
3210 /**
3211  * Apply the flow to the NIC.
3212  *
3213  * @param[in] dev
3214  *   Pointer to the Ethernet device structure.
3215  * @param[in, out] flow
3216  *   Pointer to flow structure.
3217  * @param[out] error
3218  *   Pointer to error structure.
3219  *
3220  * @return
3221  *   0 on success, a negative errno value otherwise and rte_errno is set.
3222  */
3223 static int
3224 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3225               struct rte_flow_error *error)
3226 {
3227         struct mlx5_flow_dv *dv;
3228         struct mlx5_flow *dev_flow;
3229         int n;
3230         int err;
3231
3232         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
3233                 dv = &dev_flow->dv;
3234                 n = dv->actions_n;
3235                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
3236                         dv->hrxq = mlx5_hrxq_drop_new(dev);
3237                         if (!dv->hrxq) {
3238                                 rte_flow_error_set
3239                                         (error, errno,
3240                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3241                                          "cannot get drop hash queue");
3242                                 goto error;
3243                         }
3244                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
3245                         dv->actions[n].qp = dv->hrxq->qp;
3246                         n++;
3247                 } else if (flow->actions &
3248                            (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
3249                         struct mlx5_hrxq *hrxq;
3250
3251                         hrxq = mlx5_hrxq_get(dev, flow->key,
3252                                              MLX5_RSS_HASH_KEY_LEN,
3253                                              dv->hash_fields,
3254                                              (*flow->queue),
3255                                              flow->rss.queue_num);
3256                         if (!hrxq)
3257                                 hrxq = mlx5_hrxq_new
3258                                         (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
3259                                          dv->hash_fields, (*flow->queue),
3260                                          flow->rss.queue_num,
3261                                          !!(dev_flow->layers &
3262                                             MLX5_FLOW_LAYER_TUNNEL));
3263                         if (!hrxq) {
3264                                 rte_flow_error_set
3265                                         (error, rte_errno,
3266                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3267                                          "cannot get hash queue");
3268                                 goto error;
3269                         }
3270                         dv->hrxq = hrxq;
3271                         dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
3272                         dv->actions[n].qp = hrxq->qp;
3273                         n++;
3274                 }
3275                 dv->flow =
3276                         mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
3277                                                   (void *)&dv->value, n,
3278                                                   dv->actions);
3279                 if (!dv->flow) {
3280                         rte_flow_error_set(error, errno,
3281                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3282                                            NULL,
3283                                            "hardware refuses to create flow");
3284                         goto error;
3285                 }
3286         }
3287         return 0;
3288 error:
3289         err = rte_errno; /* Save rte_errno before cleanup. */
3290         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
3291                 struct mlx5_flow_dv *dv = &dev_flow->dv;
3292                 if (dv->hrxq) {
3293                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
3294                                 mlx5_hrxq_drop_release(dev);
3295                         else
3296                                 mlx5_hrxq_release(dev, dv->hrxq);
3297                         dv->hrxq = NULL;
3298                 }
3299         }
3300         rte_errno = err; /* Restore rte_errno. */
3301         return -rte_errno;
3302 }
3303
3304 /**
3305  * Release the flow matcher.
3306  *
3307  * @param dev
3308  *   Pointer to Ethernet device.
3309  * @param flow
3310  *   Pointer to mlx5_flow.
3311  *
3312  * @return
3313  *   1 while a reference on it exists, 0 when freed.
3314  */
3315 static int
3316 flow_dv_matcher_release(struct rte_eth_dev *dev,
3317                         struct mlx5_flow *flow)
3318 {
3319         struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
3320
3321         assert(matcher->matcher_object);
3322         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
3323                 dev->data->port_id, (void *)matcher,
3324                 rte_atomic32_read(&matcher->refcnt));
3325         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
3326                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
3327                            (matcher->matcher_object));
3328                 LIST_REMOVE(matcher, next);
3329                 rte_free(matcher);
3330                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
3331                         dev->data->port_id, (void *)matcher);
3332                 return 0;
3333         }
3334         return 1;
3335 }
3336
3337 /**
3338  * Release an encap/decap resource.
3339  *
3340  * @param flow
3341  *   Pointer to mlx5_flow.
3342  *
3343  * @return
3344  *   1 while a reference on it exists, 0 when freed.
3345  */
3346 static int
3347 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
3348 {
3349         struct mlx5_flow_dv_encap_decap_resource *cache_resource =
3350                                                 flow->dv.encap_decap;
3351
3352         assert(cache_resource->verbs_action);
3353         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
3354                 (void *)cache_resource,
3355                 rte_atomic32_read(&cache_resource->refcnt));
3356         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
3357                 claim_zero(mlx5_glue->destroy_flow_action
3358                                 (cache_resource->verbs_action));
3359                 LIST_REMOVE(cache_resource, next);
3360                 rte_free(cache_resource);
3361                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
3362                         (void *)cache_resource);
3363                 return 0;
3364         }
3365         return 1;
3366 }
3367
3368 /**
3369  * Release a modify-header resource.
3370  *
3371  * @param flow
3372  *   Pointer to mlx5_flow.
3373  *
3374  * @return
3375  *   1 while a reference on it exists, 0 when freed.
3376  */
3377 static int
3378 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
3379 {
3380         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
3381                                                 flow->dv.modify_hdr;
3382
3383         assert(cache_resource->verbs_action);
3384         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
3385                 (void *)cache_resource,
3386                 rte_atomic32_read(&cache_resource->refcnt));
3387         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
3388                 claim_zero(mlx5_glue->destroy_flow_action
3389                                 (cache_resource->verbs_action));
3390                 LIST_REMOVE(cache_resource, next);
3391                 rte_free(cache_resource);
3392                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
3393                         (void *)cache_resource);
3394                 return 0;
3395         }
3396         return 1;
3397 }
3398
3399 /**
3400  * Remove the flow from the NIC but keeps it in memory.
3401  *
3402  * @param[in] dev
3403  *   Pointer to Ethernet device.
3404  * @param[in, out] flow
3405  *   Pointer to flow structure.
3406  */
3407 static void
3408 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
3409 {
3410         struct mlx5_flow_dv *dv;
3411         struct mlx5_flow *dev_flow;
3412
3413         if (!flow)
3414                 return;
3415         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
3416                 dv = &dev_flow->dv;
3417                 if (dv->flow) {
3418                         claim_zero(mlx5_glue->destroy_flow(dv->flow));
3419                         dv->flow = NULL;
3420                 }
3421                 if (dv->hrxq) {
3422                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
3423                                 mlx5_hrxq_drop_release(dev);
3424                         else
3425                                 mlx5_hrxq_release(dev, dv->hrxq);
3426                         dv->hrxq = NULL;
3427                 }
3428         }
3429 }
3430
3431 /**
3432  * Remove the flow from the NIC and the memory.
3433  *
3434  * @param[in] dev
3435  *   Pointer to the Ethernet device structure.
3436  * @param[in, out] flow
3437  *   Pointer to flow structure.
3438  */
3439 static void
3440 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3441 {
3442         struct mlx5_flow *dev_flow;
3443
3444         if (!flow)
3445                 return;
3446         flow_dv_remove(dev, flow);
3447         if (flow->counter) {
3448                 flow_dv_counter_release(flow->counter);
3449                 flow->counter = NULL;
3450         }
3451         while (!LIST_EMPTY(&flow->dev_flows)) {
3452                 dev_flow = LIST_FIRST(&flow->dev_flows);
3453                 LIST_REMOVE(dev_flow, next);
3454                 if (dev_flow->dv.matcher)
3455                         flow_dv_matcher_release(dev, dev_flow);
3456                 if (dev_flow->dv.encap_decap)
3457                         flow_dv_encap_decap_resource_release(dev_flow);
3458                 if (dev_flow->dv.modify_hdr)
3459                         flow_dv_modify_hdr_resource_release(dev_flow);
3460                 rte_free(dev_flow);
3461         }
3462 }
3463
3464 /**
3465  * Query a dv flow  rule for its statistics via devx.
3466  *
3467  * @param[in] dev
3468  *   Pointer to Ethernet device.
3469  * @param[in] flow
3470  *   Pointer to the sub flow.
3471  * @param[out] data
3472  *   data retrieved by the query.
3473  * @param[out] error
3474  *   Perform verbose error reporting if not NULL.
3475  *
3476  * @return
3477  *   0 on success, a negative errno value otherwise and rte_errno is set.
3478  */
3479 static int
3480 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
3481                     void *data, struct rte_flow_error *error)
3482 {
3483         struct priv *priv = dev->data->dev_private;
3484         struct rte_flow_query_count *qc = data;
3485         uint64_t pkts = 0;
3486         uint64_t bytes = 0;
3487         int err;
3488
3489         if (!priv->config.devx)
3490                 return rte_flow_error_set(error, ENOTSUP,
3491                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3492                                           NULL,
3493                                           "counters are not supported");
3494         if (flow->counter) {
3495                 err = mlx5_devx_cmd_flow_counter_query
3496                                                 (flow->counter->dcs,
3497                                                  qc->reset, &pkts, &bytes);
3498                 if (err)
3499                         return rte_flow_error_set
3500                                 (error, err,
3501                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3502                                  NULL,
3503                                  "cannot read counters");
3504                 qc->hits_set = 1;
3505                 qc->bytes_set = 1;
3506                 qc->hits = pkts - flow->counter->hits;
3507                 qc->bytes = bytes - flow->counter->bytes;
3508                 if (qc->reset) {
3509                         flow->counter->hits = pkts;
3510                         flow->counter->bytes = bytes;
3511                 }
3512                 return 0;
3513         }
3514         return rte_flow_error_set(error, EINVAL,
3515                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3516                                   NULL,
3517                                   "counters are not available");
3518 }
3519
3520 /**
3521  * Query a flow.
3522  *
3523  * @see rte_flow_query()
3524  * @see rte_flow_ops
3525  */
3526 static int
3527 flow_dv_query(struct rte_eth_dev *dev,
3528               struct rte_flow *flow __rte_unused,
3529               const struct rte_flow_action *actions __rte_unused,
3530               void *data __rte_unused,
3531               struct rte_flow_error *error __rte_unused)
3532 {
3533         int ret = -EINVAL;
3534
3535         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3536                 switch (actions->type) {
3537                 case RTE_FLOW_ACTION_TYPE_VOID:
3538                         break;
3539                 case RTE_FLOW_ACTION_TYPE_COUNT:
3540                         ret = flow_dv_query_count(dev, flow, data, error);
3541                         break;
3542                 default:
3543                         return rte_flow_error_set(error, ENOTSUP,
3544                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3545                                                   actions,
3546                                                   "action not supported");
3547                 }
3548         }
3549         return ret;
3550 }
3551
3552
3553 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
3554         .validate = flow_dv_validate,
3555         .prepare = flow_dv_prepare,
3556         .translate = flow_dv_translate,
3557         .apply = flow_dv_apply,
3558         .remove = flow_dv_remove,
3559         .destroy = flow_dv_destroy,
3560         .query = flow_dv_query,
3561 };
3562
3563 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */