1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
5 #include <netinet/in.h>
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
29 #include <mlx5_glue.h>
32 #include "mlx5_defs.h"
34 #include "mlx5_flow.h"
35 #include "mlx5_rxtx.h"
37 #define VERBS_SPEC_INNER(item_flags) \
38 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
41 * Get Verbs flow counter by index.
44 * Pointer to the Ethernet device structure.
46 * mlx5 flow counter index in the container.
48 * mlx5 flow counter pool in the container,
51 * A pointer to the counter, NULL otherwise.
53 static struct mlx5_flow_counter *
54 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
56 struct mlx5_flow_counter_pool **ppool)
58 struct mlx5_priv *priv = dev->data->dev_private;
59 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0,
61 struct mlx5_flow_counter_pool *pool;
64 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
68 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
72 * Create Verbs flow counter with Verbs library.
75 * Pointer to the Ethernet device structure.
76 * @param[in, out] counter
77 * mlx5 flow counter object, contains the counter id,
78 * handle of created Verbs flow counter is returned
79 * in cs field (if counters are supported).
82 * 0 On success else a negative errno value is returned
83 * and rte_errno is set.
86 flow_verbs_counter_create(struct rte_eth_dev *dev,
87 struct mlx5_flow_counter_ext *counter)
89 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
90 struct mlx5_priv *priv = dev->data->dev_private;
91 struct ibv_context *ctx = priv->sh->ctx;
92 struct ibv_counter_set_init_attr init = {
93 .counter_set_id = counter->id};
95 counter->cs = mlx5_glue->create_counter_set(ctx, &init);
101 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
102 struct mlx5_priv *priv = dev->data->dev_private;
103 struct ibv_context *ctx = priv->sh->ctx;
104 struct ibv_counters_init_attr init = {0};
105 struct ibv_counter_attach_attr attach;
108 memset(&attach, 0, sizeof(attach));
109 counter->cs = mlx5_glue->create_counters(ctx, &init);
114 attach.counter_desc = IBV_COUNTER_PACKETS;
116 ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
118 attach.counter_desc = IBV_COUNTER_BYTES;
120 ret = mlx5_glue->attach_counters
121 (counter->cs, &attach, NULL);
124 claim_zero(mlx5_glue->destroy_counters(counter->cs));
139 * Get a flow counter.
142 * Pointer to the Ethernet device structure.
144 * Indicate if this counter is shared with other flows.
146 * Counter identifier.
149 * Index to the counter, 0 otherwise and rte_errno is set.
152 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
154 struct mlx5_priv *priv = dev->data->dev_private;
155 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0,
157 struct mlx5_flow_counter_pool *pool = NULL;
158 struct mlx5_flow_counter_ext *cnt_ext = NULL;
159 struct mlx5_flow_counter *cnt = NULL;
160 uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
166 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
167 pool = cont->pools[pool_idx];
168 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
169 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
170 if (cnt_ext->shared && cnt_ext->id == id) {
172 return MLX5_MAKE_CNT_IDX(pool_idx, i);
177 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
178 pool = cont->pools[pool_idx];
181 cnt = TAILQ_FIRST(&pool->counters);
186 struct mlx5_flow_counter_pool **pools;
189 if (n_valid == cont->n) {
190 /* Resize the container pool array. */
191 size = sizeof(struct mlx5_flow_counter_pool *) *
192 (n_valid + MLX5_CNT_CONTAINER_RESIZE);
193 pools = rte_zmalloc(__func__, size, 0);
197 memcpy(pools, cont->pools,
198 sizeof(struct mlx5_flow_counter_pool *) *
200 rte_free(cont->pools);
203 cont->n += MLX5_CNT_CONTAINER_RESIZE;
205 /* Allocate memory for new pool*/
206 size = sizeof(*pool) + sizeof(*cnt_ext) *
207 MLX5_COUNTERS_PER_POOL;
208 pool = rte_calloc(__func__, 1, size, 0);
211 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
212 cnt = MLX5_POOL_GET_CNT(pool, i);
213 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
215 cnt = MLX5_POOL_GET_CNT(pool, 0);
216 cont->pools[n_valid] = pool;
218 rte_atomic16_add(&cont->n_valid, 1);
219 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
221 i = MLX5_CNT_ARRAY_IDX(pool, cnt);
222 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
224 cnt_ext->shared = shared;
225 cnt_ext->ref_cnt = 1;
228 /* Create counter with Verbs. */
229 ret = flow_verbs_counter_create(dev, cnt_ext);
231 TAILQ_REMOVE(&pool->counters, cnt, next);
232 return MLX5_MAKE_CNT_IDX(pool_idx, i);
234 /* Some error occurred in Verbs library. */
240 * Release a flow counter.
243 * Pointer to the Ethernet device structure.
245 * Index to the counter handler.
248 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
250 struct mlx5_flow_counter_pool *pool;
251 struct mlx5_flow_counter *cnt;
252 struct mlx5_flow_counter_ext *cnt_ext;
254 cnt = flow_verbs_counter_get_by_idx(dev, counter,
256 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
257 if (--cnt_ext->ref_cnt == 0) {
258 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
259 claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
261 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
262 claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs));
265 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
270 * Query a flow counter via Verbs library call.
272 * @see rte_flow_query()
276 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
277 struct rte_flow *flow, void *data,
278 struct rte_flow_error *error)
280 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
281 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
283 struct mlx5_flow_counter_pool *pool;
284 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
285 (dev, flow->counter, &pool);
286 struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT
288 struct rte_flow_query_count *qc = data;
289 uint64_t counters[2] = {0, 0};
290 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
291 struct ibv_query_counter_set_attr query_cs_attr = {
293 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
295 struct ibv_counter_set_data query_out = {
297 .outlen = 2 * sizeof(uint64_t),
299 int err = mlx5_glue->query_counter_set(&query_cs_attr,
301 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
302 int err = mlx5_glue->query_counters
303 (cnt_ext->cs, counters,
305 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
308 return rte_flow_error_set
310 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
312 "cannot read counter");
315 qc->hits = counters[0] - cnt->hits;
316 qc->bytes = counters[1] - cnt->bytes;
318 cnt->hits = counters[0];
319 cnt->bytes = counters[1];
323 return rte_flow_error_set(error, EINVAL,
324 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
326 "flow does not have counter");
330 return rte_flow_error_set(error, ENOTSUP,
331 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
333 "counters are not available");
338 * Add a verbs item specification into @p verbs.
341 * Pointer to verbs structure.
343 * Create specification.
345 * Size in bytes of the specification to copy.
348 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
349 void *src, unsigned int size)
355 MLX5_ASSERT(verbs->specs);
356 dst = (void *)(verbs->specs + verbs->size);
357 memcpy(dst, src, size);
358 ++verbs->attr.num_of_specs;
363 * Convert the @p item into a Verbs specification. This function assumes that
364 * the input is valid and that there is space to insert the requested item
367 * @param[in, out] dev_flow
368 * Pointer to dev_flow structure.
370 * Item specification.
371 * @param[in] item_flags
375 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
376 const struct rte_flow_item *item,
379 const struct rte_flow_item_eth *spec = item->spec;
380 const struct rte_flow_item_eth *mask = item->mask;
381 const unsigned int size = sizeof(struct ibv_flow_spec_eth);
382 struct ibv_flow_spec_eth eth = {
383 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
388 mask = &rte_flow_item_eth_mask;
392 memcpy(ð.val.dst_mac, spec->dst.addr_bytes,
394 memcpy(ð.val.src_mac, spec->src.addr_bytes,
396 eth.val.ether_type = spec->type;
397 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes,
399 memcpy(ð.mask.src_mac, mask->src.addr_bytes,
401 eth.mask.ether_type = mask->type;
402 /* Remove unwanted bits from values. */
403 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
404 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
405 eth.val.src_mac[i] &= eth.mask.src_mac[i];
407 eth.val.ether_type &= eth.mask.ether_type;
409 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
413 * Update the VLAN tag in the Verbs Ethernet specification.
414 * This function assumes that the input is valid and there is space to add
415 * the requested item.
417 * @param[in, out] attr
418 * Pointer to Verbs attributes structure.
420 * Verbs structure containing the VLAN information to copy.
423 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
424 struct ibv_flow_spec_eth *eth)
427 const enum ibv_flow_spec_type search = eth->type;
428 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
429 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
431 for (i = 0; i != attr->num_of_specs; ++i) {
432 if (hdr->type == search) {
433 struct ibv_flow_spec_eth *e =
434 (struct ibv_flow_spec_eth *)hdr;
436 e->val.vlan_tag = eth->val.vlan_tag;
437 e->mask.vlan_tag = eth->mask.vlan_tag;
438 e->val.ether_type = eth->val.ether_type;
439 e->mask.ether_type = eth->mask.ether_type;
442 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
447 * Convert the @p item into a Verbs specification. This function assumes that
448 * the input is valid and that there is space to insert the requested item
451 * @param[in, out] dev_flow
452 * Pointer to dev_flow structure.
454 * Item specification.
455 * @param[in] item_flags
459 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
460 const struct rte_flow_item *item,
463 const struct rte_flow_item_vlan *spec = item->spec;
464 const struct rte_flow_item_vlan *mask = item->mask;
465 unsigned int size = sizeof(struct ibv_flow_spec_eth);
466 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
467 struct ibv_flow_spec_eth eth = {
468 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
471 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
472 MLX5_FLOW_LAYER_OUTER_L2;
475 mask = &rte_flow_item_vlan_mask;
477 eth.val.vlan_tag = spec->tci;
478 eth.mask.vlan_tag = mask->tci;
479 eth.val.vlan_tag &= eth.mask.vlan_tag;
480 eth.val.ether_type = spec->inner_type;
481 eth.mask.ether_type = mask->inner_type;
482 eth.val.ether_type &= eth.mask.ether_type;
484 if (!(item_flags & l2m))
485 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
487 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð);
489 dev_flow->handle->vf_vlan.tag =
490 rte_be_to_cpu_16(spec->tci) & 0x0fff;
494 * Convert the @p item into a Verbs specification. This function assumes that
495 * the input is valid and that there is space to insert the requested item
498 * @param[in, out] dev_flow
499 * Pointer to dev_flow structure.
501 * Item specification.
502 * @param[in] item_flags
506 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
507 const struct rte_flow_item *item,
510 const struct rte_flow_item_ipv4 *spec = item->spec;
511 const struct rte_flow_item_ipv4 *mask = item->mask;
512 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
513 struct ibv_flow_spec_ipv4_ext ipv4 = {
514 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
519 mask = &rte_flow_item_ipv4_mask;
521 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
522 .src_ip = spec->hdr.src_addr,
523 .dst_ip = spec->hdr.dst_addr,
524 .proto = spec->hdr.next_proto_id,
525 .tos = spec->hdr.type_of_service,
527 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
528 .src_ip = mask->hdr.src_addr,
529 .dst_ip = mask->hdr.dst_addr,
530 .proto = mask->hdr.next_proto_id,
531 .tos = mask->hdr.type_of_service,
533 /* Remove unwanted bits from values. */
534 ipv4.val.src_ip &= ipv4.mask.src_ip;
535 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
536 ipv4.val.proto &= ipv4.mask.proto;
537 ipv4.val.tos &= ipv4.mask.tos;
539 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
543 * Convert the @p item into a Verbs specification. This function assumes that
544 * the input is valid and that there is space to insert the requested item
547 * @param[in, out] dev_flow
548 * Pointer to dev_flow structure.
550 * Item specification.
551 * @param[in] item_flags
555 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
556 const struct rte_flow_item *item,
559 const struct rte_flow_item_ipv6 *spec = item->spec;
560 const struct rte_flow_item_ipv6 *mask = item->mask;
561 unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
562 struct ibv_flow_spec_ipv6 ipv6 = {
563 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
568 mask = &rte_flow_item_ipv6_mask;
571 uint32_t vtc_flow_val;
572 uint32_t vtc_flow_mask;
574 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
575 RTE_DIM(ipv6.val.src_ip));
576 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
577 RTE_DIM(ipv6.val.dst_ip));
578 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
579 RTE_DIM(ipv6.mask.src_ip));
580 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
581 RTE_DIM(ipv6.mask.dst_ip));
582 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
583 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
584 ipv6.val.flow_label =
585 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
586 RTE_IPV6_HDR_FL_SHIFT);
587 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
588 RTE_IPV6_HDR_TC_SHIFT;
589 ipv6.val.next_hdr = spec->hdr.proto;
590 ipv6.mask.flow_label =
591 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
592 RTE_IPV6_HDR_FL_SHIFT);
593 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
594 RTE_IPV6_HDR_TC_SHIFT;
595 ipv6.mask.next_hdr = mask->hdr.proto;
596 /* Remove unwanted bits from values. */
597 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
598 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
599 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
601 ipv6.val.flow_label &= ipv6.mask.flow_label;
602 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
603 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
605 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
609 * Convert the @p item into a Verbs specification. This function assumes that
610 * the input is valid and that there is space to insert the requested item
613 * @param[in, out] dev_flow
614 * Pointer to dev_flow structure.
616 * Item specification.
617 * @param[in] item_flags
621 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
622 const struct rte_flow_item *item,
623 uint64_t item_flags __rte_unused)
625 const struct rte_flow_item_tcp *spec = item->spec;
626 const struct rte_flow_item_tcp *mask = item->mask;
627 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
628 struct ibv_flow_spec_tcp_udp tcp = {
629 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
634 mask = &rte_flow_item_tcp_mask;
636 tcp.val.dst_port = spec->hdr.dst_port;
637 tcp.val.src_port = spec->hdr.src_port;
638 tcp.mask.dst_port = mask->hdr.dst_port;
639 tcp.mask.src_port = mask->hdr.src_port;
640 /* Remove unwanted bits from values. */
641 tcp.val.src_port &= tcp.mask.src_port;
642 tcp.val.dst_port &= tcp.mask.dst_port;
644 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
648 * Convert the @p item into a Verbs specification. This function assumes that
649 * the input is valid and that there is space to insert the requested item
652 * @param[in, out] dev_flow
653 * Pointer to dev_flow structure.
655 * Item specification.
656 * @param[in] item_flags
660 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
661 const struct rte_flow_item *item,
662 uint64_t item_flags __rte_unused)
664 const struct rte_flow_item_udp *spec = item->spec;
665 const struct rte_flow_item_udp *mask = item->mask;
666 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
667 struct ibv_flow_spec_tcp_udp udp = {
668 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
673 mask = &rte_flow_item_udp_mask;
675 udp.val.dst_port = spec->hdr.dst_port;
676 udp.val.src_port = spec->hdr.src_port;
677 udp.mask.dst_port = mask->hdr.dst_port;
678 udp.mask.src_port = mask->hdr.src_port;
679 /* Remove unwanted bits from values. */
680 udp.val.src_port &= udp.mask.src_port;
681 udp.val.dst_port &= udp.mask.dst_port;
684 while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
686 if (!(udp.val.dst_port & udp.mask.dst_port)) {
687 switch ((item)->type) {
688 case RTE_FLOW_ITEM_TYPE_VXLAN:
689 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
690 udp.mask.dst_port = 0xffff;
692 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
693 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
694 udp.mask.dst_port = 0xffff;
696 case RTE_FLOW_ITEM_TYPE_MPLS:
697 udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
698 udp.mask.dst_port = 0xffff;
705 flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
709 * Convert the @p item into a Verbs specification. This function assumes that
710 * the input is valid and that there is space to insert the requested item
713 * @param[in, out] dev_flow
714 * Pointer to dev_flow structure.
716 * Item specification.
717 * @param[in] item_flags
721 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
722 const struct rte_flow_item *item,
723 uint64_t item_flags __rte_unused)
725 const struct rte_flow_item_vxlan *spec = item->spec;
726 const struct rte_flow_item_vxlan *mask = item->mask;
727 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
728 struct ibv_flow_spec_tunnel vxlan = {
729 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
735 } id = { .vlan_id = 0, };
738 mask = &rte_flow_item_vxlan_mask;
740 memcpy(&id.vni[1], spec->vni, 3);
741 vxlan.val.tunnel_id = id.vlan_id;
742 memcpy(&id.vni[1], mask->vni, 3);
743 vxlan.mask.tunnel_id = id.vlan_id;
744 /* Remove unwanted bits from values. */
745 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
747 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
751 * Convert the @p item into a Verbs specification. This function assumes that
752 * the input is valid and that there is space to insert the requested item
755 * @param[in, out] dev_flow
756 * Pointer to dev_flow structure.
758 * Item specification.
759 * @param[in] item_flags
763 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
764 const struct rte_flow_item *item,
765 uint64_t item_flags __rte_unused)
767 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
768 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
769 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
770 struct ibv_flow_spec_tunnel vxlan_gpe = {
771 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
777 } id = { .vlan_id = 0, };
780 mask = &rte_flow_item_vxlan_gpe_mask;
782 memcpy(&id.vni[1], spec->vni, 3);
783 vxlan_gpe.val.tunnel_id = id.vlan_id;
784 memcpy(&id.vni[1], mask->vni, 3);
785 vxlan_gpe.mask.tunnel_id = id.vlan_id;
786 /* Remove unwanted bits from values. */
787 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
789 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
793 * Update the protocol in Verbs IPv4/IPv6 spec.
795 * @param[in, out] attr
796 * Pointer to Verbs attributes structure.
798 * Specification type to search in order to update the IP protocol.
799 * @param[in] protocol
800 * Protocol value to set if none is present in the specification.
803 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
804 enum ibv_flow_spec_type search,
808 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
809 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
813 for (i = 0; i != attr->num_of_specs; ++i) {
814 if (hdr->type == search) {
816 struct ibv_flow_spec_ipv4_ext *ipv4;
817 struct ibv_flow_spec_ipv6 *ipv6;
821 case IBV_FLOW_SPEC_IPV4_EXT:
822 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
823 if (!ip.ipv4->val.proto) {
824 ip.ipv4->val.proto = protocol;
825 ip.ipv4->mask.proto = 0xff;
828 case IBV_FLOW_SPEC_IPV6:
829 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
830 if (!ip.ipv6->val.next_hdr) {
831 ip.ipv6->val.next_hdr = protocol;
832 ip.ipv6->mask.next_hdr = 0xff;
840 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
845 * Convert the @p item into a Verbs specification. This function assumes that
846 * the input is valid and that there is space to insert the requested item
849 * @param[in, out] dev_flow
850 * Pointer to dev_flow structure.
852 * Item specification.
853 * @param[in] item_flags
857 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
858 const struct rte_flow_item *item __rte_unused,
861 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
862 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
863 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
864 struct ibv_flow_spec_tunnel tunnel = {
865 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
869 const struct rte_flow_item_gre *spec = item->spec;
870 const struct rte_flow_item_gre *mask = item->mask;
871 unsigned int size = sizeof(struct ibv_flow_spec_gre);
872 struct ibv_flow_spec_gre tunnel = {
873 .type = IBV_FLOW_SPEC_GRE,
878 mask = &rte_flow_item_gre_mask;
880 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
881 tunnel.val.protocol = spec->protocol;
882 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
883 tunnel.mask.protocol = mask->protocol;
884 /* Remove unwanted bits from values. */
885 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
886 tunnel.val.protocol &= tunnel.mask.protocol;
887 tunnel.val.key &= tunnel.mask.key;
890 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
891 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
892 IBV_FLOW_SPEC_IPV4_EXT,
895 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
898 flow_verbs_spec_add(verbs, &tunnel, size);
902 * Convert the @p action into a Verbs specification. This function assumes that
903 * the input is valid and that there is space to insert the requested action
904 * into the flow. This function also return the action that was added.
906 * @param[in, out] dev_flow
907 * Pointer to dev_flow structure.
909 * Item specification.
910 * @param[in] item_flags
914 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
915 const struct rte_flow_item *item __rte_unused,
916 uint64_t item_flags __rte_unused)
918 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
919 const struct rte_flow_item_mpls *spec = item->spec;
920 const struct rte_flow_item_mpls *mask = item->mask;
921 unsigned int size = sizeof(struct ibv_flow_spec_mpls);
922 struct ibv_flow_spec_mpls mpls = {
923 .type = IBV_FLOW_SPEC_MPLS,
928 mask = &rte_flow_item_mpls_mask;
930 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
931 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
932 /* Remove unwanted bits from values. */
933 mpls.val.label &= mpls.mask.label;
935 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
940 * Convert the @p action into a Verbs specification. This function assumes that
941 * the input is valid and that there is space to insert the requested action
944 * @param[in] dev_flow
945 * Pointer to mlx5_flow.
947 * Action configuration.
950 flow_verbs_translate_action_drop
951 (struct mlx5_flow *dev_flow,
952 const struct rte_flow_action *action __rte_unused)
954 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
955 struct ibv_flow_spec_action_drop drop = {
956 .type = IBV_FLOW_SPEC_ACTION_DROP,
960 flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
964 * Convert the @p action into a Verbs specification. This function assumes that
965 * the input is valid and that there is space to insert the requested action
968 * @param[in] rss_desc
969 * Pointer to mlx5_flow_rss_desc.
971 * Action configuration.
974 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
975 const struct rte_flow_action *action)
977 const struct rte_flow_action_queue *queue = action->conf;
979 rss_desc->queue[0] = queue->index;
980 rss_desc->queue_num = 1;
984 * Convert the @p action into a Verbs specification. This function assumes that
985 * the input is valid and that there is space to insert the requested action
988 * @param[in] rss_desc
989 * Pointer to mlx5_flow_rss_desc.
991 * Action configuration.
994 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
995 const struct rte_flow_action *action)
997 const struct rte_flow_action_rss *rss = action->conf;
998 const uint8_t *rss_key;
1000 memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1001 rss_desc->queue_num = rss->queue_num;
1002 /* NULL RSS key indicates default RSS key. */
1003 rss_key = !rss->key ? rss_hash_default_key : rss->key;
1004 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1006 * rss->level and rss.types should be set in advance when expanding
1012 * Convert the @p action into a Verbs specification. This function assumes that
1013 * the input is valid and that there is space to insert the requested action
1016 * @param[in] dev_flow
1017 * Pointer to mlx5_flow.
1019 * Action configuration.
1022 flow_verbs_translate_action_flag
1023 (struct mlx5_flow *dev_flow,
1024 const struct rte_flow_action *action __rte_unused)
1026 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1027 struct ibv_flow_spec_action_tag tag = {
1028 .type = IBV_FLOW_SPEC_ACTION_TAG,
1030 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1033 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1037 * Convert the @p action into a Verbs specification. This function assumes that
1038 * the input is valid and that there is space to insert the requested action
1041 * @param[in] dev_flow
1042 * Pointer to mlx5_flow.
1044 * Action configuration.
1047 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1048 const struct rte_flow_action *action)
1050 const struct rte_flow_action_mark *mark = action->conf;
1051 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1052 struct ibv_flow_spec_action_tag tag = {
1053 .type = IBV_FLOW_SPEC_ACTION_TAG,
1055 .tag_id = mlx5_flow_mark_set(mark->id),
1058 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1062 * Convert the @p action into a Verbs specification. This function assumes that
1063 * the input is valid and that there is space to insert the requested action
1067 * Pointer to the Ethernet device structure.
1069 * Action configuration.
1070 * @param[in] dev_flow
1071 * Pointer to mlx5_flow.
1073 * Pointer to error structure.
1076 * 0 On success else a negative errno value is returned and rte_errno is set.
1079 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1080 const struct rte_flow_action *action,
1081 struct rte_eth_dev *dev,
1082 struct rte_flow_error *error)
1084 const struct rte_flow_action_count *count = action->conf;
1085 struct rte_flow *flow = dev_flow->flow;
1086 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1087 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1088 struct mlx5_flow_counter_pool *pool;
1089 struct mlx5_flow_counter *cnt = NULL;
1090 struct mlx5_flow_counter_ext *cnt_ext;
1091 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1092 struct ibv_flow_spec_counter_action counter = {
1093 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1098 if (!flow->counter) {
1099 flow->counter = flow_verbs_counter_new(dev, count->shared,
1102 return rte_flow_error_set(error, rte_errno,
1103 RTE_FLOW_ERROR_TYPE_ACTION,
1105 "cannot get counter"
1108 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1109 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1110 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1111 counter.counter_set_handle = cnt_ext->cs->handle;
1112 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1113 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1114 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1115 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1116 counter.counters = cnt_ext->cs;
1117 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1123 * Internal validation function. For validating both actions and items.
1126 * Pointer to the Ethernet device structure.
1128 * Pointer to the flow attributes.
1130 * Pointer to the list of items.
1131 * @param[in] actions
1132 * Pointer to the list of actions.
1133 * @param[in] external
1134 * This flow rule is created by request external to PMD.
1135 * @param[in] hairpin
1136 * Number of hairpin TX actions, 0 means classic flow.
1138 * Pointer to the error structure.
1141 * 0 on success, a negative errno value otherwise and rte_errno is set.
1144 flow_verbs_validate(struct rte_eth_dev *dev,
1145 const struct rte_flow_attr *attr,
1146 const struct rte_flow_item items[],
1147 const struct rte_flow_action actions[],
1148 bool external __rte_unused,
1149 int hairpin __rte_unused,
1150 struct rte_flow_error *error)
1153 uint64_t action_flags = 0;
1154 uint64_t item_flags = 0;
1155 uint64_t last_item = 0;
1156 uint8_t next_protocol = 0xff;
1157 uint16_t ether_type = 0;
1161 ret = mlx5_flow_validate_attributes(dev, attr, error);
1164 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1165 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1168 switch (items->type) {
1169 case RTE_FLOW_ITEM_TYPE_VOID:
1171 case RTE_FLOW_ITEM_TYPE_ETH:
1172 ret = mlx5_flow_validate_item_eth(items, item_flags,
1176 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1177 MLX5_FLOW_LAYER_OUTER_L2;
1178 if (items->mask != NULL && items->spec != NULL) {
1180 ((const struct rte_flow_item_eth *)
1183 ((const struct rte_flow_item_eth *)
1185 ether_type = rte_be_to_cpu_16(ether_type);
1190 case RTE_FLOW_ITEM_TYPE_VLAN:
1191 ret = mlx5_flow_validate_item_vlan(items, item_flags,
1195 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1196 MLX5_FLOW_LAYER_INNER_VLAN) :
1197 (MLX5_FLOW_LAYER_OUTER_L2 |
1198 MLX5_FLOW_LAYER_OUTER_VLAN);
1199 if (items->mask != NULL && items->spec != NULL) {
1201 ((const struct rte_flow_item_vlan *)
1202 items->spec)->inner_type;
1204 ((const struct rte_flow_item_vlan *)
1205 items->mask)->inner_type;
1206 ether_type = rte_be_to_cpu_16(ether_type);
1211 case RTE_FLOW_ITEM_TYPE_IPV4:
1212 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1218 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1219 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1220 if (items->mask != NULL &&
1221 ((const struct rte_flow_item_ipv4 *)
1222 items->mask)->hdr.next_proto_id) {
1224 ((const struct rte_flow_item_ipv4 *)
1225 (items->spec))->hdr.next_proto_id;
1227 ((const struct rte_flow_item_ipv4 *)
1228 (items->mask))->hdr.next_proto_id;
1230 /* Reset for inner layer. */
1231 next_protocol = 0xff;
1234 case RTE_FLOW_ITEM_TYPE_IPV6:
1235 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1241 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1242 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1243 if (items->mask != NULL &&
1244 ((const struct rte_flow_item_ipv6 *)
1245 items->mask)->hdr.proto) {
1247 ((const struct rte_flow_item_ipv6 *)
1248 items->spec)->hdr.proto;
1250 ((const struct rte_flow_item_ipv6 *)
1251 items->mask)->hdr.proto;
1253 /* Reset for inner layer. */
1254 next_protocol = 0xff;
1257 case RTE_FLOW_ITEM_TYPE_UDP:
1258 ret = mlx5_flow_validate_item_udp(items, item_flags,
1263 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1264 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1266 case RTE_FLOW_ITEM_TYPE_TCP:
1267 ret = mlx5_flow_validate_item_tcp
1270 &rte_flow_item_tcp_mask,
1274 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1275 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1277 case RTE_FLOW_ITEM_TYPE_VXLAN:
1278 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1282 last_item = MLX5_FLOW_LAYER_VXLAN;
1284 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1285 ret = mlx5_flow_validate_item_vxlan_gpe(items,
1290 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1292 case RTE_FLOW_ITEM_TYPE_GRE:
1293 ret = mlx5_flow_validate_item_gre(items, item_flags,
1294 next_protocol, error);
1297 last_item = MLX5_FLOW_LAYER_GRE;
1299 case RTE_FLOW_ITEM_TYPE_MPLS:
1300 ret = mlx5_flow_validate_item_mpls(dev, items,
1305 last_item = MLX5_FLOW_LAYER_MPLS;
1308 return rte_flow_error_set(error, ENOTSUP,
1309 RTE_FLOW_ERROR_TYPE_ITEM,
1310 NULL, "item not supported");
1312 item_flags |= last_item;
1314 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1315 switch (actions->type) {
1316 case RTE_FLOW_ACTION_TYPE_VOID:
1318 case RTE_FLOW_ACTION_TYPE_FLAG:
1319 ret = mlx5_flow_validate_action_flag(action_flags,
1324 action_flags |= MLX5_FLOW_ACTION_FLAG;
1326 case RTE_FLOW_ACTION_TYPE_MARK:
1327 ret = mlx5_flow_validate_action_mark(actions,
1333 action_flags |= MLX5_FLOW_ACTION_MARK;
1335 case RTE_FLOW_ACTION_TYPE_DROP:
1336 ret = mlx5_flow_validate_action_drop(action_flags,
1341 action_flags |= MLX5_FLOW_ACTION_DROP;
1343 case RTE_FLOW_ACTION_TYPE_QUEUE:
1344 ret = mlx5_flow_validate_action_queue(actions,
1350 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1352 case RTE_FLOW_ACTION_TYPE_RSS:
1353 ret = mlx5_flow_validate_action_rss(actions,
1359 action_flags |= MLX5_FLOW_ACTION_RSS;
1361 case RTE_FLOW_ACTION_TYPE_COUNT:
1362 ret = mlx5_flow_validate_action_count(dev, attr, error);
1365 action_flags |= MLX5_FLOW_ACTION_COUNT;
1368 return rte_flow_error_set(error, ENOTSUP,
1369 RTE_FLOW_ERROR_TYPE_ACTION,
1371 "action not supported");
1375 * Validate the drop action mutual exclusion with other actions.
1376 * Drop action is mutually-exclusive with any other action, except for
1379 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1380 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1381 return rte_flow_error_set(error, EINVAL,
1382 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1383 "Drop action is mutually-exclusive "
1384 "with any other action, except for "
1386 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1387 return rte_flow_error_set(error, EINVAL,
1388 RTE_FLOW_ERROR_TYPE_ACTION, actions,
1389 "no fate action is found");
1394 * Calculate the required bytes that are needed for the action part of the verbs
1397 * @param[in] actions
1398 * Pointer to the list of actions.
1401 * The size of the memory needed for all actions.
1404 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1408 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1409 switch (actions->type) {
1410 case RTE_FLOW_ACTION_TYPE_VOID:
1412 case RTE_FLOW_ACTION_TYPE_FLAG:
1413 size += sizeof(struct ibv_flow_spec_action_tag);
1415 case RTE_FLOW_ACTION_TYPE_MARK:
1416 size += sizeof(struct ibv_flow_spec_action_tag);
1418 case RTE_FLOW_ACTION_TYPE_DROP:
1419 size += sizeof(struct ibv_flow_spec_action_drop);
1421 case RTE_FLOW_ACTION_TYPE_QUEUE:
1423 case RTE_FLOW_ACTION_TYPE_RSS:
1425 case RTE_FLOW_ACTION_TYPE_COUNT:
1426 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1427 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1428 size += sizeof(struct ibv_flow_spec_counter_action);
1439 * Calculate the required bytes that are needed for the item part of the verbs
1443 * Pointer to the list of items.
1446 * The size of the memory needed for all items.
1449 flow_verbs_get_items_size(const struct rte_flow_item items[])
1453 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1454 switch (items->type) {
1455 case RTE_FLOW_ITEM_TYPE_VOID:
1457 case RTE_FLOW_ITEM_TYPE_ETH:
1458 size += sizeof(struct ibv_flow_spec_eth);
1460 case RTE_FLOW_ITEM_TYPE_VLAN:
1461 size += sizeof(struct ibv_flow_spec_eth);
1463 case RTE_FLOW_ITEM_TYPE_IPV4:
1464 size += sizeof(struct ibv_flow_spec_ipv4_ext);
1466 case RTE_FLOW_ITEM_TYPE_IPV6:
1467 size += sizeof(struct ibv_flow_spec_ipv6);
1469 case RTE_FLOW_ITEM_TYPE_UDP:
1470 size += sizeof(struct ibv_flow_spec_tcp_udp);
1472 case RTE_FLOW_ITEM_TYPE_TCP:
1473 size += sizeof(struct ibv_flow_spec_tcp_udp);
1475 case RTE_FLOW_ITEM_TYPE_VXLAN:
1476 size += sizeof(struct ibv_flow_spec_tunnel);
1478 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1479 size += sizeof(struct ibv_flow_spec_tunnel);
1481 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1482 case RTE_FLOW_ITEM_TYPE_GRE:
1483 size += sizeof(struct ibv_flow_spec_gre);
1485 case RTE_FLOW_ITEM_TYPE_MPLS:
1486 size += sizeof(struct ibv_flow_spec_mpls);
1489 case RTE_FLOW_ITEM_TYPE_GRE:
1490 size += sizeof(struct ibv_flow_spec_tunnel);
1501 * Internal preparation function. Allocate mlx5_flow with the required size.
1502 * The required size is calculate based on the actions and items. This function
1503 * also returns the detected actions and items for later use.
1506 * Pointer to Ethernet device.
1508 * Pointer to the flow attributes.
1510 * Pointer to the list of items.
1511 * @param[in] actions
1512 * Pointer to the list of actions.
1514 * Pointer to the error structure.
1517 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1520 static struct mlx5_flow *
1521 flow_verbs_prepare(struct rte_eth_dev *dev,
1522 const struct rte_flow_attr *attr __rte_unused,
1523 const struct rte_flow_item items[],
1524 const struct rte_flow_action actions[],
1525 struct rte_flow_error *error)
1528 uint32_t handle_idx = 0;
1529 struct mlx5_flow *dev_flow;
1530 struct mlx5_flow_handle *dev_handle;
1531 struct mlx5_priv *priv = dev->data->dev_private;
1533 size += flow_verbs_get_actions_size(actions);
1534 size += flow_verbs_get_items_size(items);
1535 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1536 rte_flow_error_set(error, E2BIG,
1537 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1538 "Verbs spec/action size too large");
1541 /* In case of corrupting the memory. */
1542 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1543 rte_flow_error_set(error, ENOSPC,
1544 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1545 "not free temporary device flow");
1548 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1551 rte_flow_error_set(error, ENOMEM,
1552 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1553 "not enough memory to create flow handle");
1556 /* No multi-thread supporting. */
1557 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1558 dev_flow->handle = dev_handle;
1559 dev_flow->handle_idx = handle_idx;
1560 /* Memcpy is used, only size needs to be cleared to 0. */
1561 dev_flow->verbs.size = 0;
1562 dev_flow->verbs.attr.num_of_specs = 0;
1563 dev_flow->ingress = attr->ingress;
1564 dev_flow->hash_fields = 0;
1565 /* Need to set transfer attribute: not supported in Verbs mode. */
1570 * Fill the flow with verb spec.
1573 * Pointer to Ethernet device.
1574 * @param[in, out] dev_flow
1575 * Pointer to the mlx5 flow.
1577 * Pointer to the flow attributes.
1579 * Pointer to the list of items.
1580 * @param[in] actions
1581 * Pointer to the list of actions.
1583 * Pointer to the error structure.
1586 * 0 on success, else a negative errno value otherwise and rte_errno is set.
1589 flow_verbs_translate(struct rte_eth_dev *dev,
1590 struct mlx5_flow *dev_flow,
1591 const struct rte_flow_attr *attr,
1592 const struct rte_flow_item items[],
1593 const struct rte_flow_action actions[],
1594 struct rte_flow_error *error)
1596 uint64_t item_flags = 0;
1597 uint64_t action_flags = 0;
1598 uint64_t priority = attr->priority;
1599 uint32_t subpriority = 0;
1600 struct mlx5_priv *priv = dev->data->dev_private;
1601 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
1603 [!!priv->flow_nested_idx];
1605 if (priority == MLX5_FLOW_PRIO_RSVD)
1606 priority = priv->config.flow_prio - 1;
1607 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1610 switch (actions->type) {
1611 case RTE_FLOW_ACTION_TYPE_VOID:
1613 case RTE_FLOW_ACTION_TYPE_FLAG:
1614 flow_verbs_translate_action_flag(dev_flow, actions);
1615 action_flags |= MLX5_FLOW_ACTION_FLAG;
1616 dev_flow->handle->mark = 1;
1618 case RTE_FLOW_ACTION_TYPE_MARK:
1619 flow_verbs_translate_action_mark(dev_flow, actions);
1620 action_flags |= MLX5_FLOW_ACTION_MARK;
1621 dev_flow->handle->mark = 1;
1623 case RTE_FLOW_ACTION_TYPE_DROP:
1624 flow_verbs_translate_action_drop(dev_flow, actions);
1625 action_flags |= MLX5_FLOW_ACTION_DROP;
1626 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1628 case RTE_FLOW_ACTION_TYPE_QUEUE:
1629 flow_verbs_translate_action_queue(rss_desc, actions);
1630 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1631 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1633 case RTE_FLOW_ACTION_TYPE_RSS:
1634 flow_verbs_translate_action_rss(rss_desc, actions);
1635 action_flags |= MLX5_FLOW_ACTION_RSS;
1636 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1638 case RTE_FLOW_ACTION_TYPE_COUNT:
1639 ret = flow_verbs_translate_action_count(dev_flow,
1644 action_flags |= MLX5_FLOW_ACTION_COUNT;
1647 return rte_flow_error_set(error, ENOTSUP,
1648 RTE_FLOW_ERROR_TYPE_ACTION,
1650 "action not supported");
1653 dev_flow->act_flags = action_flags;
1654 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1655 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1657 switch (items->type) {
1658 case RTE_FLOW_ITEM_TYPE_VOID:
1660 case RTE_FLOW_ITEM_TYPE_ETH:
1661 flow_verbs_translate_item_eth(dev_flow, items,
1663 subpriority = MLX5_PRIORITY_MAP_L2;
1664 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1665 MLX5_FLOW_LAYER_OUTER_L2;
1667 case RTE_FLOW_ITEM_TYPE_VLAN:
1668 flow_verbs_translate_item_vlan(dev_flow, items,
1670 subpriority = MLX5_PRIORITY_MAP_L2;
1671 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1672 MLX5_FLOW_LAYER_INNER_VLAN) :
1673 (MLX5_FLOW_LAYER_OUTER_L2 |
1674 MLX5_FLOW_LAYER_OUTER_VLAN);
1676 case RTE_FLOW_ITEM_TYPE_IPV4:
1677 flow_verbs_translate_item_ipv4(dev_flow, items,
1679 subpriority = MLX5_PRIORITY_MAP_L3;
1680 dev_flow->hash_fields |=
1681 mlx5_flow_hashfields_adjust
1683 MLX5_IPV4_LAYER_TYPES,
1684 MLX5_IPV4_IBV_RX_HASH);
1685 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1686 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1688 case RTE_FLOW_ITEM_TYPE_IPV6:
1689 flow_verbs_translate_item_ipv6(dev_flow, items,
1691 subpriority = MLX5_PRIORITY_MAP_L3;
1692 dev_flow->hash_fields |=
1693 mlx5_flow_hashfields_adjust
1695 MLX5_IPV6_LAYER_TYPES,
1696 MLX5_IPV6_IBV_RX_HASH);
1697 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1698 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1700 case RTE_FLOW_ITEM_TYPE_TCP:
1701 flow_verbs_translate_item_tcp(dev_flow, items,
1703 subpriority = MLX5_PRIORITY_MAP_L4;
1704 dev_flow->hash_fields |=
1705 mlx5_flow_hashfields_adjust
1706 (rss_desc, tunnel, ETH_RSS_TCP,
1707 (IBV_RX_HASH_SRC_PORT_TCP |
1708 IBV_RX_HASH_DST_PORT_TCP));
1709 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1710 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1712 case RTE_FLOW_ITEM_TYPE_UDP:
1713 flow_verbs_translate_item_udp(dev_flow, items,
1715 subpriority = MLX5_PRIORITY_MAP_L4;
1716 dev_flow->hash_fields |=
1717 mlx5_flow_hashfields_adjust
1718 (rss_desc, tunnel, ETH_RSS_UDP,
1719 (IBV_RX_HASH_SRC_PORT_UDP |
1720 IBV_RX_HASH_DST_PORT_UDP));
1721 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1722 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1724 case RTE_FLOW_ITEM_TYPE_VXLAN:
1725 flow_verbs_translate_item_vxlan(dev_flow, items,
1727 subpriority = MLX5_PRIORITY_MAP_L2;
1728 item_flags |= MLX5_FLOW_LAYER_VXLAN;
1730 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1731 flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1733 subpriority = MLX5_PRIORITY_MAP_L2;
1734 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1736 case RTE_FLOW_ITEM_TYPE_GRE:
1737 flow_verbs_translate_item_gre(dev_flow, items,
1739 subpriority = MLX5_PRIORITY_MAP_L2;
1740 item_flags |= MLX5_FLOW_LAYER_GRE;
1742 case RTE_FLOW_ITEM_TYPE_MPLS:
1743 flow_verbs_translate_item_mpls(dev_flow, items,
1745 subpriority = MLX5_PRIORITY_MAP_L2;
1746 item_flags |= MLX5_FLOW_LAYER_MPLS;
1749 return rte_flow_error_set(error, ENOTSUP,
1750 RTE_FLOW_ERROR_TYPE_ITEM,
1752 "item not supported");
1755 dev_flow->handle->layers = item_flags;
1756 /* Other members of attr will be ignored. */
1757 dev_flow->verbs.attr.priority =
1758 mlx5_flow_adjust_priority(dev, priority, subpriority);
1759 dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port;
1764 * Remove the flow from the NIC but keeps it in memory.
1767 * Pointer to the Ethernet device structure.
1768 * @param[in, out] flow
1769 * Pointer to flow structure.
1772 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1774 struct mlx5_priv *priv = dev->data->dev_private;
1775 struct mlx5_flow_handle *handle;
1776 uint32_t handle_idx;
1780 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1781 handle_idx, handle, next) {
1782 if (handle->ib_flow) {
1783 claim_zero(mlx5_glue->destroy_flow(handle->ib_flow));
1784 handle->ib_flow = NULL;
1786 /* hrxq is union, don't touch it only the flag is set. */
1787 if (handle->rix_hrxq) {
1788 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1789 mlx5_hrxq_drop_release(dev);
1790 handle->rix_hrxq = 0;
1791 } else if (handle->fate_action ==
1792 MLX5_FLOW_FATE_QUEUE) {
1793 mlx5_hrxq_release(dev, handle->rix_hrxq);
1794 handle->rix_hrxq = 0;
1797 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1798 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1803 * Remove the flow from the NIC and the memory.
1806 * Pointer to the Ethernet device structure.
1807 * @param[in, out] flow
1808 * Pointer to flow structure.
1811 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1813 struct mlx5_priv *priv = dev->data->dev_private;
1814 struct mlx5_flow_handle *handle;
1818 flow_verbs_remove(dev, flow);
1819 while (flow->dev_handles) {
1820 uint32_t tmp_idx = flow->dev_handles;
1822 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1826 flow->dev_handles = handle->next.next;
1827 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1830 if (flow->counter) {
1831 flow_verbs_counter_release(dev, flow->counter);
1837 * Apply the flow to the NIC.
1840 * Pointer to the Ethernet device structure.
1841 * @param[in, out] flow
1842 * Pointer to flow structure.
1844 * Pointer to error structure.
1847 * 0 on success, a negative errno value otherwise and rte_errno is set.
1850 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1851 struct rte_flow_error *error)
1853 struct mlx5_priv *priv = dev->data->dev_private;
1854 struct mlx5_flow_handle *handle;
1855 struct mlx5_flow *dev_flow;
1856 struct mlx5_hrxq *hrxq;
1857 uint32_t dev_handles;
1861 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
1862 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1863 handle = dev_flow->handle;
1864 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1865 hrxq = mlx5_hrxq_drop_new(dev);
1869 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1870 "cannot get drop hash queue");
1875 struct mlx5_flow_rss_desc *rss_desc =
1876 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
1877 [!!priv->flow_nested_idx];
1879 MLX5_ASSERT(rss_desc->queue_num);
1880 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
1881 MLX5_RSS_HASH_KEY_LEN,
1882 dev_flow->hash_fields,
1884 rss_desc->queue_num);
1886 hrxq_idx = mlx5_hrxq_new(dev, rss_desc->key,
1887 MLX5_RSS_HASH_KEY_LEN,
1888 dev_flow->hash_fields,
1890 rss_desc->queue_num,
1892 MLX5_FLOW_LAYER_TUNNEL));
1893 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1898 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1899 "cannot get hash queue");
1902 handle->rix_hrxq = hrxq_idx;
1905 handle->ib_flow = mlx5_glue->create_flow(hrxq->qp,
1906 &dev_flow->verbs.attr);
1907 if (!handle->ib_flow) {
1908 rte_flow_error_set(error, errno,
1909 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1911 "hardware refuses to create flow");
1914 if (priv->vmwa_context &&
1915 handle->vf_vlan.tag && !handle->vf_vlan.created) {
1917 * The rule contains the VLAN pattern.
1918 * For VF we are going to create VLAN
1919 * interface to make hypervisor set correct
1920 * e-Switch vport context.
1922 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
1927 err = rte_errno; /* Save rte_errno before cleanup. */
1928 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1929 dev_handles, handle, next) {
1930 /* hrxq is union, don't touch it only the flag is set. */
1931 if (handle->rix_hrxq) {
1932 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1933 mlx5_hrxq_drop_release(dev);
1934 handle->rix_hrxq = 0;
1935 } else if (handle->fate_action ==
1936 MLX5_FLOW_FATE_QUEUE) {
1937 mlx5_hrxq_release(dev, handle->rix_hrxq);
1938 handle->rix_hrxq = 0;
1941 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1942 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1944 rte_errno = err; /* Restore rte_errno. */
1951 * @see rte_flow_query()
1955 flow_verbs_query(struct rte_eth_dev *dev,
1956 struct rte_flow *flow,
1957 const struct rte_flow_action *actions,
1959 struct rte_flow_error *error)
1963 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1964 switch (actions->type) {
1965 case RTE_FLOW_ACTION_TYPE_VOID:
1967 case RTE_FLOW_ACTION_TYPE_COUNT:
1968 ret = flow_verbs_counter_query(dev, flow, data, error);
1971 return rte_flow_error_set(error, ENOTSUP,
1972 RTE_FLOW_ERROR_TYPE_ACTION,
1974 "action not supported");
1980 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1981 .validate = flow_verbs_validate,
1982 .prepare = flow_verbs_prepare,
1983 .translate = flow_verbs_translate,
1984 .apply = flow_verbs_apply,
1985 .remove = flow_verbs_remove,
1986 .destroy = flow_verbs_destroy,
1987 .query = flow_verbs_query,