1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
5 #include <netinet/in.h>
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
29 #include <mlx5_glue.h>
31 #include <mlx5_malloc.h>
33 #include "mlx5_defs.h"
35 #include "mlx5_flow.h"
36 #include "mlx5_rxtx.h"
38 #define VERBS_SPEC_INNER(item_flags) \
39 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
42 * Get Verbs flow counter by index.
45 * Pointer to the Ethernet device structure.
47 * mlx5 flow counter index in the container.
49 * mlx5 flow counter pool in the container,
52 * A pointer to the counter, NULL otherwise.
54 static struct mlx5_flow_counter *
55 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
57 struct mlx5_flow_counter_pool **ppool)
59 struct mlx5_priv *priv = dev->data->dev_private;
60 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
61 struct mlx5_flow_counter_pool *pool;
64 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
68 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
72 * Create Verbs flow counter with Verbs library.
75 * Pointer to the Ethernet device structure.
76 * @param[in, out] counter
77 * mlx5 flow counter object, contains the counter id,
78 * handle of created Verbs flow counter is returned
79 * in cs field (if counters are supported).
82 * 0 On success else a negative errno value is returned
83 * and rte_errno is set.
86 flow_verbs_counter_create(struct rte_eth_dev *dev,
87 struct mlx5_flow_counter_ext *counter)
89 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
90 struct mlx5_priv *priv = dev->data->dev_private;
91 struct ibv_context *ctx = priv->sh->ctx;
92 struct ibv_counter_set_init_attr init = {
93 .counter_set_id = counter->id};
95 counter->cs = mlx5_glue->create_counter_set(ctx, &init);
101 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
102 struct mlx5_priv *priv = dev->data->dev_private;
103 struct ibv_context *ctx = priv->sh->ctx;
104 struct ibv_counters_init_attr init = {0};
105 struct ibv_counter_attach_attr attach;
108 memset(&attach, 0, sizeof(attach));
109 counter->cs = mlx5_glue->create_counters(ctx, &init);
114 attach.counter_desc = IBV_COUNTER_PACKETS;
116 ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
118 attach.counter_desc = IBV_COUNTER_BYTES;
120 ret = mlx5_glue->attach_counters
121 (counter->cs, &attach, NULL);
124 claim_zero(mlx5_glue->destroy_counters(counter->cs));
139 * Get a flow counter.
142 * Pointer to the Ethernet device structure.
144 * Indicate if this counter is shared with other flows.
146 * Counter identifier.
149 * Index to the counter, 0 otherwise and rte_errno is set.
152 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
154 struct mlx5_priv *priv = dev->data->dev_private;
155 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
156 struct mlx5_flow_counter_pool *pool = NULL;
157 struct mlx5_flow_counter_ext *cnt_ext = NULL;
158 struct mlx5_flow_counter *cnt = NULL;
159 uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
165 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
166 pool = cont->pools[pool_idx];
167 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
168 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
169 if (cnt_ext->shared && cnt_ext->id == id) {
171 return MLX5_MAKE_CNT_IDX(pool_idx, i);
176 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
177 pool = cont->pools[pool_idx];
180 cnt = TAILQ_FIRST(&pool->counters[0]);
185 struct mlx5_flow_counter_pool **pools;
188 if (n_valid == cont->n) {
189 /* Resize the container pool array. */
190 size = sizeof(struct mlx5_flow_counter_pool *) *
191 (n_valid + MLX5_CNT_CONTAINER_RESIZE);
192 pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
197 memcpy(pools, cont->pools,
198 sizeof(struct mlx5_flow_counter_pool *) *
200 mlx5_free(cont->pools);
203 cont->n += MLX5_CNT_CONTAINER_RESIZE;
205 /* Allocate memory for new pool*/
206 size = sizeof(*pool) + (sizeof(*cnt_ext) + sizeof(*cnt)) *
207 MLX5_COUNTERS_PER_POOL;
208 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
211 pool->type |= CNT_POOL_TYPE_EXT;
212 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
213 cnt = MLX5_POOL_GET_CNT(pool, i);
214 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
216 cnt = MLX5_POOL_GET_CNT(pool, 0);
217 cont->pools[n_valid] = pool;
219 rte_atomic16_add(&cont->n_valid, 1);
220 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
222 i = MLX5_CNT_ARRAY_IDX(pool, cnt);
223 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
225 cnt_ext->shared = shared;
226 cnt_ext->ref_cnt = 1;
229 /* Create counter with Verbs. */
230 ret = flow_verbs_counter_create(dev, cnt_ext);
232 TAILQ_REMOVE(&pool->counters[0], cnt, next);
233 return MLX5_MAKE_CNT_IDX(pool_idx, i);
235 /* Some error occurred in Verbs library. */
241 * Release a flow counter.
244 * Pointer to the Ethernet device structure.
246 * Index to the counter handler.
249 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
251 struct mlx5_flow_counter_pool *pool;
252 struct mlx5_flow_counter *cnt;
253 struct mlx5_flow_counter_ext *cnt_ext;
255 cnt = flow_verbs_counter_get_by_idx(dev, counter,
257 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
258 if (--cnt_ext->ref_cnt == 0) {
259 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
260 claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
262 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
263 claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs));
266 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
271 * Query a flow counter via Verbs library call.
273 * @see rte_flow_query()
277 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
278 struct rte_flow *flow, void *data,
279 struct rte_flow_error *error)
281 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
282 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
284 struct mlx5_flow_counter_pool *pool;
285 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
286 (dev, flow->counter, &pool);
287 struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT
289 struct rte_flow_query_count *qc = data;
290 uint64_t counters[2] = {0, 0};
291 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
292 struct ibv_query_counter_set_attr query_cs_attr = {
294 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
296 struct ibv_counter_set_data query_out = {
298 .outlen = 2 * sizeof(uint64_t),
300 int err = mlx5_glue->query_counter_set(&query_cs_attr,
302 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
303 int err = mlx5_glue->query_counters
304 (cnt_ext->cs, counters,
306 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
309 return rte_flow_error_set
311 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
313 "cannot read counter");
316 qc->hits = counters[0] - cnt->hits;
317 qc->bytes = counters[1] - cnt->bytes;
319 cnt->hits = counters[0];
320 cnt->bytes = counters[1];
324 return rte_flow_error_set(error, EINVAL,
325 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
327 "flow does not have counter");
331 return rte_flow_error_set(error, ENOTSUP,
332 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
334 "counters are not available");
339 * Add a verbs item specification into @p verbs.
342 * Pointer to verbs structure.
344 * Create specification.
346 * Size in bytes of the specification to copy.
349 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
350 void *src, unsigned int size)
356 MLX5_ASSERT(verbs->specs);
357 dst = (void *)(verbs->specs + verbs->size);
358 memcpy(dst, src, size);
359 ++verbs->attr.num_of_specs;
364 * Convert the @p item into a Verbs specification. This function assumes that
365 * the input is valid and that there is space to insert the requested item
368 * @param[in, out] dev_flow
369 * Pointer to dev_flow structure.
371 * Item specification.
372 * @param[in] item_flags
376 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
377 const struct rte_flow_item *item,
380 const struct rte_flow_item_eth *spec = item->spec;
381 const struct rte_flow_item_eth *mask = item->mask;
382 const unsigned int size = sizeof(struct ibv_flow_spec_eth);
383 struct ibv_flow_spec_eth eth = {
384 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
389 mask = &rte_flow_item_eth_mask;
393 memcpy(ð.val.dst_mac, spec->dst.addr_bytes,
395 memcpy(ð.val.src_mac, spec->src.addr_bytes,
397 eth.val.ether_type = spec->type;
398 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes,
400 memcpy(ð.mask.src_mac, mask->src.addr_bytes,
402 eth.mask.ether_type = mask->type;
403 /* Remove unwanted bits from values. */
404 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
405 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
406 eth.val.src_mac[i] &= eth.mask.src_mac[i];
408 eth.val.ether_type &= eth.mask.ether_type;
410 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
414 * Update the VLAN tag in the Verbs Ethernet specification.
415 * This function assumes that the input is valid and there is space to add
416 * the requested item.
418 * @param[in, out] attr
419 * Pointer to Verbs attributes structure.
421 * Verbs structure containing the VLAN information to copy.
424 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
425 struct ibv_flow_spec_eth *eth)
428 const enum ibv_flow_spec_type search = eth->type;
429 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
430 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
432 for (i = 0; i != attr->num_of_specs; ++i) {
433 if (hdr->type == search) {
434 struct ibv_flow_spec_eth *e =
435 (struct ibv_flow_spec_eth *)hdr;
437 e->val.vlan_tag = eth->val.vlan_tag;
438 e->mask.vlan_tag = eth->mask.vlan_tag;
439 e->val.ether_type = eth->val.ether_type;
440 e->mask.ether_type = eth->mask.ether_type;
443 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
448 * Convert the @p item into a Verbs specification. This function assumes that
449 * the input is valid and that there is space to insert the requested item
452 * @param[in, out] dev_flow
453 * Pointer to dev_flow structure.
455 * Item specification.
456 * @param[in] item_flags
460 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
461 const struct rte_flow_item *item,
464 const struct rte_flow_item_vlan *spec = item->spec;
465 const struct rte_flow_item_vlan *mask = item->mask;
466 unsigned int size = sizeof(struct ibv_flow_spec_eth);
467 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
468 struct ibv_flow_spec_eth eth = {
469 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
472 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
473 MLX5_FLOW_LAYER_OUTER_L2;
476 mask = &rte_flow_item_vlan_mask;
478 eth.val.vlan_tag = spec->tci;
479 eth.mask.vlan_tag = mask->tci;
480 eth.val.vlan_tag &= eth.mask.vlan_tag;
481 eth.val.ether_type = spec->inner_type;
482 eth.mask.ether_type = mask->inner_type;
483 eth.val.ether_type &= eth.mask.ether_type;
485 if (!(item_flags & l2m))
486 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
488 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð);
490 dev_flow->handle->vf_vlan.tag =
491 rte_be_to_cpu_16(spec->tci) & 0x0fff;
495 * Convert the @p item into a Verbs specification. This function assumes that
496 * the input is valid and that there is space to insert the requested item
499 * @param[in, out] dev_flow
500 * Pointer to dev_flow structure.
502 * Item specification.
503 * @param[in] item_flags
507 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
508 const struct rte_flow_item *item,
511 const struct rte_flow_item_ipv4 *spec = item->spec;
512 const struct rte_flow_item_ipv4 *mask = item->mask;
513 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
514 struct ibv_flow_spec_ipv4_ext ipv4 = {
515 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
520 mask = &rte_flow_item_ipv4_mask;
522 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
523 .src_ip = spec->hdr.src_addr,
524 .dst_ip = spec->hdr.dst_addr,
525 .proto = spec->hdr.next_proto_id,
526 .tos = spec->hdr.type_of_service,
528 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
529 .src_ip = mask->hdr.src_addr,
530 .dst_ip = mask->hdr.dst_addr,
531 .proto = mask->hdr.next_proto_id,
532 .tos = mask->hdr.type_of_service,
534 /* Remove unwanted bits from values. */
535 ipv4.val.src_ip &= ipv4.mask.src_ip;
536 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
537 ipv4.val.proto &= ipv4.mask.proto;
538 ipv4.val.tos &= ipv4.mask.tos;
540 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
544 * Convert the @p item into a Verbs specification. This function assumes that
545 * the input is valid and that there is space to insert the requested item
548 * @param[in, out] dev_flow
549 * Pointer to dev_flow structure.
551 * Item specification.
552 * @param[in] item_flags
556 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
557 const struct rte_flow_item *item,
560 const struct rte_flow_item_ipv6 *spec = item->spec;
561 const struct rte_flow_item_ipv6 *mask = item->mask;
562 unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
563 struct ibv_flow_spec_ipv6 ipv6 = {
564 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
569 mask = &rte_flow_item_ipv6_mask;
572 uint32_t vtc_flow_val;
573 uint32_t vtc_flow_mask;
575 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
576 RTE_DIM(ipv6.val.src_ip));
577 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
578 RTE_DIM(ipv6.val.dst_ip));
579 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
580 RTE_DIM(ipv6.mask.src_ip));
581 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
582 RTE_DIM(ipv6.mask.dst_ip));
583 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
584 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
585 ipv6.val.flow_label =
586 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
587 RTE_IPV6_HDR_FL_SHIFT);
588 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
589 RTE_IPV6_HDR_TC_SHIFT;
590 ipv6.val.next_hdr = spec->hdr.proto;
591 ipv6.mask.flow_label =
592 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
593 RTE_IPV6_HDR_FL_SHIFT);
594 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
595 RTE_IPV6_HDR_TC_SHIFT;
596 ipv6.mask.next_hdr = mask->hdr.proto;
597 /* Remove unwanted bits from values. */
598 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
599 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
600 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
602 ipv6.val.flow_label &= ipv6.mask.flow_label;
603 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
604 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
606 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
610 * Convert the @p item into a Verbs specification. This function assumes that
611 * the input is valid and that there is space to insert the requested item
614 * @param[in, out] dev_flow
615 * Pointer to dev_flow structure.
617 * Item specification.
618 * @param[in] item_flags
622 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
623 const struct rte_flow_item *item,
624 uint64_t item_flags __rte_unused)
626 const struct rte_flow_item_tcp *spec = item->spec;
627 const struct rte_flow_item_tcp *mask = item->mask;
628 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
629 struct ibv_flow_spec_tcp_udp tcp = {
630 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
635 mask = &rte_flow_item_tcp_mask;
637 tcp.val.dst_port = spec->hdr.dst_port;
638 tcp.val.src_port = spec->hdr.src_port;
639 tcp.mask.dst_port = mask->hdr.dst_port;
640 tcp.mask.src_port = mask->hdr.src_port;
641 /* Remove unwanted bits from values. */
642 tcp.val.src_port &= tcp.mask.src_port;
643 tcp.val.dst_port &= tcp.mask.dst_port;
645 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
649 * Convert the @p item into a Verbs specification. This function assumes that
650 * the input is valid and that there is space to insert the requested item
653 * @param[in, out] dev_flow
654 * Pointer to dev_flow structure.
656 * Item specification.
657 * @param[in] item_flags
661 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
662 const struct rte_flow_item *item,
663 uint64_t item_flags __rte_unused)
665 const struct rte_flow_item_udp *spec = item->spec;
666 const struct rte_flow_item_udp *mask = item->mask;
667 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
668 struct ibv_flow_spec_tcp_udp udp = {
669 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
674 mask = &rte_flow_item_udp_mask;
676 udp.val.dst_port = spec->hdr.dst_port;
677 udp.val.src_port = spec->hdr.src_port;
678 udp.mask.dst_port = mask->hdr.dst_port;
679 udp.mask.src_port = mask->hdr.src_port;
680 /* Remove unwanted bits from values. */
681 udp.val.src_port &= udp.mask.src_port;
682 udp.val.dst_port &= udp.mask.dst_port;
685 while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
687 if (!(udp.val.dst_port & udp.mask.dst_port)) {
688 switch ((item)->type) {
689 case RTE_FLOW_ITEM_TYPE_VXLAN:
690 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
691 udp.mask.dst_port = 0xffff;
693 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
694 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
695 udp.mask.dst_port = 0xffff;
697 case RTE_FLOW_ITEM_TYPE_MPLS:
698 udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
699 udp.mask.dst_port = 0xffff;
706 flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
710 * Convert the @p item into a Verbs specification. This function assumes that
711 * the input is valid and that there is space to insert the requested item
714 * @param[in, out] dev_flow
715 * Pointer to dev_flow structure.
717 * Item specification.
718 * @param[in] item_flags
722 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
723 const struct rte_flow_item *item,
724 uint64_t item_flags __rte_unused)
726 const struct rte_flow_item_vxlan *spec = item->spec;
727 const struct rte_flow_item_vxlan *mask = item->mask;
728 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
729 struct ibv_flow_spec_tunnel vxlan = {
730 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
736 } id = { .vlan_id = 0, };
739 mask = &rte_flow_item_vxlan_mask;
741 memcpy(&id.vni[1], spec->vni, 3);
742 vxlan.val.tunnel_id = id.vlan_id;
743 memcpy(&id.vni[1], mask->vni, 3);
744 vxlan.mask.tunnel_id = id.vlan_id;
745 /* Remove unwanted bits from values. */
746 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
748 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
752 * Convert the @p item into a Verbs specification. This function assumes that
753 * the input is valid and that there is space to insert the requested item
756 * @param[in, out] dev_flow
757 * Pointer to dev_flow structure.
759 * Item specification.
760 * @param[in] item_flags
764 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
765 const struct rte_flow_item *item,
766 uint64_t item_flags __rte_unused)
768 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
769 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
770 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
771 struct ibv_flow_spec_tunnel vxlan_gpe = {
772 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
778 } id = { .vlan_id = 0, };
781 mask = &rte_flow_item_vxlan_gpe_mask;
783 memcpy(&id.vni[1], spec->vni, 3);
784 vxlan_gpe.val.tunnel_id = id.vlan_id;
785 memcpy(&id.vni[1], mask->vni, 3);
786 vxlan_gpe.mask.tunnel_id = id.vlan_id;
787 /* Remove unwanted bits from values. */
788 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
790 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
794 * Update the protocol in Verbs IPv4/IPv6 spec.
796 * @param[in, out] attr
797 * Pointer to Verbs attributes structure.
799 * Specification type to search in order to update the IP protocol.
800 * @param[in] protocol
801 * Protocol value to set if none is present in the specification.
804 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
805 enum ibv_flow_spec_type search,
809 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
810 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
814 for (i = 0; i != attr->num_of_specs; ++i) {
815 if (hdr->type == search) {
817 struct ibv_flow_spec_ipv4_ext *ipv4;
818 struct ibv_flow_spec_ipv6 *ipv6;
822 case IBV_FLOW_SPEC_IPV4_EXT:
823 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
824 if (!ip.ipv4->val.proto) {
825 ip.ipv4->val.proto = protocol;
826 ip.ipv4->mask.proto = 0xff;
829 case IBV_FLOW_SPEC_IPV6:
830 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
831 if (!ip.ipv6->val.next_hdr) {
832 ip.ipv6->val.next_hdr = protocol;
833 ip.ipv6->mask.next_hdr = 0xff;
841 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
846 * Convert the @p item into a Verbs specification. This function assumes that
847 * the input is valid and that there is space to insert the requested item
850 * @param[in, out] dev_flow
851 * Pointer to dev_flow structure.
853 * Item specification.
854 * @param[in] item_flags
858 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
859 const struct rte_flow_item *item __rte_unused,
862 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
863 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
864 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
865 struct ibv_flow_spec_tunnel tunnel = {
866 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
870 const struct rte_flow_item_gre *spec = item->spec;
871 const struct rte_flow_item_gre *mask = item->mask;
872 unsigned int size = sizeof(struct ibv_flow_spec_gre);
873 struct ibv_flow_spec_gre tunnel = {
874 .type = IBV_FLOW_SPEC_GRE,
879 mask = &rte_flow_item_gre_mask;
881 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
882 tunnel.val.protocol = spec->protocol;
883 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
884 tunnel.mask.protocol = mask->protocol;
885 /* Remove unwanted bits from values. */
886 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
887 tunnel.val.protocol &= tunnel.mask.protocol;
888 tunnel.val.key &= tunnel.mask.key;
891 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
892 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
893 IBV_FLOW_SPEC_IPV4_EXT,
896 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
899 flow_verbs_spec_add(verbs, &tunnel, size);
903 * Convert the @p action into a Verbs specification. This function assumes that
904 * the input is valid and that there is space to insert the requested action
905 * into the flow. This function also return the action that was added.
907 * @param[in, out] dev_flow
908 * Pointer to dev_flow structure.
910 * Item specification.
911 * @param[in] item_flags
915 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
916 const struct rte_flow_item *item __rte_unused,
917 uint64_t item_flags __rte_unused)
919 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
920 const struct rte_flow_item_mpls *spec = item->spec;
921 const struct rte_flow_item_mpls *mask = item->mask;
922 unsigned int size = sizeof(struct ibv_flow_spec_mpls);
923 struct ibv_flow_spec_mpls mpls = {
924 .type = IBV_FLOW_SPEC_MPLS,
929 mask = &rte_flow_item_mpls_mask;
931 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
932 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
933 /* Remove unwanted bits from values. */
934 mpls.val.label &= mpls.mask.label;
936 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
941 * Convert the @p action into a Verbs specification. This function assumes that
942 * the input is valid and that there is space to insert the requested action
945 * @param[in] dev_flow
946 * Pointer to mlx5_flow.
948 * Action configuration.
951 flow_verbs_translate_action_drop
952 (struct mlx5_flow *dev_flow,
953 const struct rte_flow_action *action __rte_unused)
955 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
956 struct ibv_flow_spec_action_drop drop = {
957 .type = IBV_FLOW_SPEC_ACTION_DROP,
961 flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
965 * Convert the @p action into a Verbs specification. This function assumes that
966 * the input is valid and that there is space to insert the requested action
969 * @param[in] rss_desc
970 * Pointer to mlx5_flow_rss_desc.
972 * Action configuration.
975 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
976 const struct rte_flow_action *action)
978 const struct rte_flow_action_queue *queue = action->conf;
980 rss_desc->queue[0] = queue->index;
981 rss_desc->queue_num = 1;
985 * Convert the @p action into a Verbs specification. This function assumes that
986 * the input is valid and that there is space to insert the requested action
989 * @param[in] rss_desc
990 * Pointer to mlx5_flow_rss_desc.
992 * Action configuration.
995 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
996 const struct rte_flow_action *action)
998 const struct rte_flow_action_rss *rss = action->conf;
999 const uint8_t *rss_key;
1001 memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1002 rss_desc->queue_num = rss->queue_num;
1003 /* NULL RSS key indicates default RSS key. */
1004 rss_key = !rss->key ? rss_hash_default_key : rss->key;
1005 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1007 * rss->level and rss.types should be set in advance when expanding
1013 * Convert the @p action into a Verbs specification. This function assumes that
1014 * the input is valid and that there is space to insert the requested action
1017 * @param[in] dev_flow
1018 * Pointer to mlx5_flow.
1020 * Action configuration.
1023 flow_verbs_translate_action_flag
1024 (struct mlx5_flow *dev_flow,
1025 const struct rte_flow_action *action __rte_unused)
1027 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1028 struct ibv_flow_spec_action_tag tag = {
1029 .type = IBV_FLOW_SPEC_ACTION_TAG,
1031 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1034 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1038 * Convert the @p action into a Verbs specification. This function assumes that
1039 * the input is valid and that there is space to insert the requested action
1042 * @param[in] dev_flow
1043 * Pointer to mlx5_flow.
1045 * Action configuration.
1048 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1049 const struct rte_flow_action *action)
1051 const struct rte_flow_action_mark *mark = action->conf;
1052 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1053 struct ibv_flow_spec_action_tag tag = {
1054 .type = IBV_FLOW_SPEC_ACTION_TAG,
1056 .tag_id = mlx5_flow_mark_set(mark->id),
1059 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1063 * Convert the @p action into a Verbs specification. This function assumes that
1064 * the input is valid and that there is space to insert the requested action
1068 * Pointer to the Ethernet device structure.
1070 * Action configuration.
1071 * @param[in] dev_flow
1072 * Pointer to mlx5_flow.
1074 * Pointer to error structure.
1077 * 0 On success else a negative errno value is returned and rte_errno is set.
1080 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1081 const struct rte_flow_action *action,
1082 struct rte_eth_dev *dev,
1083 struct rte_flow_error *error)
1085 const struct rte_flow_action_count *count = action->conf;
1086 struct rte_flow *flow = dev_flow->flow;
1087 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1088 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1089 struct mlx5_flow_counter_pool *pool;
1090 struct mlx5_flow_counter *cnt = NULL;
1091 struct mlx5_flow_counter_ext *cnt_ext;
1092 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1093 struct ibv_flow_spec_counter_action counter = {
1094 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1099 if (!flow->counter) {
1100 flow->counter = flow_verbs_counter_new(dev, count->shared,
1103 return rte_flow_error_set(error, rte_errno,
1104 RTE_FLOW_ERROR_TYPE_ACTION,
1106 "cannot get counter"
1109 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1110 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1111 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1112 counter.counter_set_handle = cnt_ext->cs->handle;
1113 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1114 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1115 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1116 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1117 counter.counters = cnt_ext->cs;
1118 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1124 * Internal validation function. For validating both actions and items.
1127 * Pointer to the Ethernet device structure.
1129 * Pointer to the flow attributes.
1131 * Pointer to the list of items.
1132 * @param[in] actions
1133 * Pointer to the list of actions.
1134 * @param[in] external
1135 * This flow rule is created by request external to PMD.
1136 * @param[in] hairpin
1137 * Number of hairpin TX actions, 0 means classic flow.
1139 * Pointer to the error structure.
1142 * 0 on success, a negative errno value otherwise and rte_errno is set.
1145 flow_verbs_validate(struct rte_eth_dev *dev,
1146 const struct rte_flow_attr *attr,
1147 const struct rte_flow_item items[],
1148 const struct rte_flow_action actions[],
1149 bool external __rte_unused,
1150 int hairpin __rte_unused,
1151 struct rte_flow_error *error)
1154 uint64_t action_flags = 0;
1155 uint64_t item_flags = 0;
1156 uint64_t last_item = 0;
1157 uint8_t next_protocol = 0xff;
1158 uint16_t ether_type = 0;
1162 ret = mlx5_flow_validate_attributes(dev, attr, error);
1165 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1166 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1169 switch (items->type) {
1170 case RTE_FLOW_ITEM_TYPE_VOID:
1172 case RTE_FLOW_ITEM_TYPE_ETH:
1173 ret = mlx5_flow_validate_item_eth(items, item_flags,
1177 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1178 MLX5_FLOW_LAYER_OUTER_L2;
1179 if (items->mask != NULL && items->spec != NULL) {
1181 ((const struct rte_flow_item_eth *)
1184 ((const struct rte_flow_item_eth *)
1186 ether_type = rte_be_to_cpu_16(ether_type);
1191 case RTE_FLOW_ITEM_TYPE_VLAN:
1192 ret = mlx5_flow_validate_item_vlan(items, item_flags,
1196 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1197 MLX5_FLOW_LAYER_INNER_VLAN) :
1198 (MLX5_FLOW_LAYER_OUTER_L2 |
1199 MLX5_FLOW_LAYER_OUTER_VLAN);
1200 if (items->mask != NULL && items->spec != NULL) {
1202 ((const struct rte_flow_item_vlan *)
1203 items->spec)->inner_type;
1205 ((const struct rte_flow_item_vlan *)
1206 items->mask)->inner_type;
1207 ether_type = rte_be_to_cpu_16(ether_type);
1212 case RTE_FLOW_ITEM_TYPE_IPV4:
1213 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1219 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1220 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1221 if (items->mask != NULL &&
1222 ((const struct rte_flow_item_ipv4 *)
1223 items->mask)->hdr.next_proto_id) {
1225 ((const struct rte_flow_item_ipv4 *)
1226 (items->spec))->hdr.next_proto_id;
1228 ((const struct rte_flow_item_ipv4 *)
1229 (items->mask))->hdr.next_proto_id;
1231 /* Reset for inner layer. */
1232 next_protocol = 0xff;
1235 case RTE_FLOW_ITEM_TYPE_IPV6:
1236 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1242 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1243 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1244 if (items->mask != NULL &&
1245 ((const struct rte_flow_item_ipv6 *)
1246 items->mask)->hdr.proto) {
1248 ((const struct rte_flow_item_ipv6 *)
1249 items->spec)->hdr.proto;
1251 ((const struct rte_flow_item_ipv6 *)
1252 items->mask)->hdr.proto;
1254 /* Reset for inner layer. */
1255 next_protocol = 0xff;
1258 case RTE_FLOW_ITEM_TYPE_UDP:
1259 ret = mlx5_flow_validate_item_udp(items, item_flags,
1264 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1265 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1267 case RTE_FLOW_ITEM_TYPE_TCP:
1268 ret = mlx5_flow_validate_item_tcp
1271 &rte_flow_item_tcp_mask,
1275 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1276 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1278 case RTE_FLOW_ITEM_TYPE_VXLAN:
1279 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1283 last_item = MLX5_FLOW_LAYER_VXLAN;
1285 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1286 ret = mlx5_flow_validate_item_vxlan_gpe(items,
1291 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1293 case RTE_FLOW_ITEM_TYPE_GRE:
1294 ret = mlx5_flow_validate_item_gre(items, item_flags,
1295 next_protocol, error);
1298 last_item = MLX5_FLOW_LAYER_GRE;
1300 case RTE_FLOW_ITEM_TYPE_MPLS:
1301 ret = mlx5_flow_validate_item_mpls(dev, items,
1306 last_item = MLX5_FLOW_LAYER_MPLS;
1309 return rte_flow_error_set(error, ENOTSUP,
1310 RTE_FLOW_ERROR_TYPE_ITEM,
1311 NULL, "item not supported");
1313 item_flags |= last_item;
1315 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1316 switch (actions->type) {
1317 case RTE_FLOW_ACTION_TYPE_VOID:
1319 case RTE_FLOW_ACTION_TYPE_FLAG:
1320 ret = mlx5_flow_validate_action_flag(action_flags,
1325 action_flags |= MLX5_FLOW_ACTION_FLAG;
1327 case RTE_FLOW_ACTION_TYPE_MARK:
1328 ret = mlx5_flow_validate_action_mark(actions,
1334 action_flags |= MLX5_FLOW_ACTION_MARK;
1336 case RTE_FLOW_ACTION_TYPE_DROP:
1337 ret = mlx5_flow_validate_action_drop(action_flags,
1342 action_flags |= MLX5_FLOW_ACTION_DROP;
1344 case RTE_FLOW_ACTION_TYPE_QUEUE:
1345 ret = mlx5_flow_validate_action_queue(actions,
1351 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1353 case RTE_FLOW_ACTION_TYPE_RSS:
1354 ret = mlx5_flow_validate_action_rss(actions,
1360 action_flags |= MLX5_FLOW_ACTION_RSS;
1362 case RTE_FLOW_ACTION_TYPE_COUNT:
1363 ret = mlx5_flow_validate_action_count(dev, attr, error);
1366 action_flags |= MLX5_FLOW_ACTION_COUNT;
1369 return rte_flow_error_set(error, ENOTSUP,
1370 RTE_FLOW_ERROR_TYPE_ACTION,
1372 "action not supported");
1376 * Validate the drop action mutual exclusion with other actions.
1377 * Drop action is mutually-exclusive with any other action, except for
1380 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1381 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1382 return rte_flow_error_set(error, EINVAL,
1383 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1384 "Drop action is mutually-exclusive "
1385 "with any other action, except for "
1387 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1388 return rte_flow_error_set(error, EINVAL,
1389 RTE_FLOW_ERROR_TYPE_ACTION, actions,
1390 "no fate action is found");
1395 * Calculate the required bytes that are needed for the action part of the verbs
1398 * @param[in] actions
1399 * Pointer to the list of actions.
1402 * The size of the memory needed for all actions.
1405 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1409 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1410 switch (actions->type) {
1411 case RTE_FLOW_ACTION_TYPE_VOID:
1413 case RTE_FLOW_ACTION_TYPE_FLAG:
1414 size += sizeof(struct ibv_flow_spec_action_tag);
1416 case RTE_FLOW_ACTION_TYPE_MARK:
1417 size += sizeof(struct ibv_flow_spec_action_tag);
1419 case RTE_FLOW_ACTION_TYPE_DROP:
1420 size += sizeof(struct ibv_flow_spec_action_drop);
1422 case RTE_FLOW_ACTION_TYPE_QUEUE:
1424 case RTE_FLOW_ACTION_TYPE_RSS:
1426 case RTE_FLOW_ACTION_TYPE_COUNT:
1427 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1428 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1429 size += sizeof(struct ibv_flow_spec_counter_action);
1440 * Calculate the required bytes that are needed for the item part of the verbs
1444 * Pointer to the list of items.
1447 * The size of the memory needed for all items.
1450 flow_verbs_get_items_size(const struct rte_flow_item items[])
1454 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1455 switch (items->type) {
1456 case RTE_FLOW_ITEM_TYPE_VOID:
1458 case RTE_FLOW_ITEM_TYPE_ETH:
1459 size += sizeof(struct ibv_flow_spec_eth);
1461 case RTE_FLOW_ITEM_TYPE_VLAN:
1462 size += sizeof(struct ibv_flow_spec_eth);
1464 case RTE_FLOW_ITEM_TYPE_IPV4:
1465 size += sizeof(struct ibv_flow_spec_ipv4_ext);
1467 case RTE_FLOW_ITEM_TYPE_IPV6:
1468 size += sizeof(struct ibv_flow_spec_ipv6);
1470 case RTE_FLOW_ITEM_TYPE_UDP:
1471 size += sizeof(struct ibv_flow_spec_tcp_udp);
1473 case RTE_FLOW_ITEM_TYPE_TCP:
1474 size += sizeof(struct ibv_flow_spec_tcp_udp);
1476 case RTE_FLOW_ITEM_TYPE_VXLAN:
1477 size += sizeof(struct ibv_flow_spec_tunnel);
1479 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1480 size += sizeof(struct ibv_flow_spec_tunnel);
1482 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1483 case RTE_FLOW_ITEM_TYPE_GRE:
1484 size += sizeof(struct ibv_flow_spec_gre);
1486 case RTE_FLOW_ITEM_TYPE_MPLS:
1487 size += sizeof(struct ibv_flow_spec_mpls);
1490 case RTE_FLOW_ITEM_TYPE_GRE:
1491 size += sizeof(struct ibv_flow_spec_tunnel);
1502 * Internal preparation function. Allocate mlx5_flow with the required size.
1503 * The required size is calculate based on the actions and items. This function
1504 * also returns the detected actions and items for later use.
1507 * Pointer to Ethernet device.
1509 * Pointer to the flow attributes.
1511 * Pointer to the list of items.
1512 * @param[in] actions
1513 * Pointer to the list of actions.
1515 * Pointer to the error structure.
1518 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1521 static struct mlx5_flow *
1522 flow_verbs_prepare(struct rte_eth_dev *dev,
1523 const struct rte_flow_attr *attr __rte_unused,
1524 const struct rte_flow_item items[],
1525 const struct rte_flow_action actions[],
1526 struct rte_flow_error *error)
1529 uint32_t handle_idx = 0;
1530 struct mlx5_flow *dev_flow;
1531 struct mlx5_flow_handle *dev_handle;
1532 struct mlx5_priv *priv = dev->data->dev_private;
1534 size += flow_verbs_get_actions_size(actions);
1535 size += flow_verbs_get_items_size(items);
1536 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1537 rte_flow_error_set(error, E2BIG,
1538 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1539 "Verbs spec/action size too large");
1542 /* In case of corrupting the memory. */
1543 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1544 rte_flow_error_set(error, ENOSPC,
1545 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1546 "not free temporary device flow");
1549 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1552 rte_flow_error_set(error, ENOMEM,
1553 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1554 "not enough memory to create flow handle");
1557 /* No multi-thread supporting. */
1558 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1559 dev_flow->handle = dev_handle;
1560 dev_flow->handle_idx = handle_idx;
1561 /* Memcpy is used, only size needs to be cleared to 0. */
1562 dev_flow->verbs.size = 0;
1563 dev_flow->verbs.attr.num_of_specs = 0;
1564 dev_flow->ingress = attr->ingress;
1565 dev_flow->hash_fields = 0;
1566 /* Need to set transfer attribute: not supported in Verbs mode. */
1571 * Fill the flow with verb spec.
1574 * Pointer to Ethernet device.
1575 * @param[in, out] dev_flow
1576 * Pointer to the mlx5 flow.
1578 * Pointer to the flow attributes.
1580 * Pointer to the list of items.
1581 * @param[in] actions
1582 * Pointer to the list of actions.
1584 * Pointer to the error structure.
1587 * 0 on success, else a negative errno value otherwise and rte_errno is set.
1590 flow_verbs_translate(struct rte_eth_dev *dev,
1591 struct mlx5_flow *dev_flow,
1592 const struct rte_flow_attr *attr,
1593 const struct rte_flow_item items[],
1594 const struct rte_flow_action actions[],
1595 struct rte_flow_error *error)
1597 uint64_t item_flags = 0;
1598 uint64_t action_flags = 0;
1599 uint64_t priority = attr->priority;
1600 uint32_t subpriority = 0;
1601 struct mlx5_priv *priv = dev->data->dev_private;
1602 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
1604 [!!priv->flow_nested_idx];
1606 if (priority == MLX5_FLOW_PRIO_RSVD)
1607 priority = priv->config.flow_prio - 1;
1608 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1611 switch (actions->type) {
1612 case RTE_FLOW_ACTION_TYPE_VOID:
1614 case RTE_FLOW_ACTION_TYPE_FLAG:
1615 flow_verbs_translate_action_flag(dev_flow, actions);
1616 action_flags |= MLX5_FLOW_ACTION_FLAG;
1617 dev_flow->handle->mark = 1;
1619 case RTE_FLOW_ACTION_TYPE_MARK:
1620 flow_verbs_translate_action_mark(dev_flow, actions);
1621 action_flags |= MLX5_FLOW_ACTION_MARK;
1622 dev_flow->handle->mark = 1;
1624 case RTE_FLOW_ACTION_TYPE_DROP:
1625 flow_verbs_translate_action_drop(dev_flow, actions);
1626 action_flags |= MLX5_FLOW_ACTION_DROP;
1627 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1629 case RTE_FLOW_ACTION_TYPE_QUEUE:
1630 flow_verbs_translate_action_queue(rss_desc, actions);
1631 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1632 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1634 case RTE_FLOW_ACTION_TYPE_RSS:
1635 flow_verbs_translate_action_rss(rss_desc, actions);
1636 action_flags |= MLX5_FLOW_ACTION_RSS;
1637 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1639 case RTE_FLOW_ACTION_TYPE_COUNT:
1640 ret = flow_verbs_translate_action_count(dev_flow,
1645 action_flags |= MLX5_FLOW_ACTION_COUNT;
1648 return rte_flow_error_set(error, ENOTSUP,
1649 RTE_FLOW_ERROR_TYPE_ACTION,
1651 "action not supported");
1654 dev_flow->act_flags = action_flags;
1655 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1656 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1658 switch (items->type) {
1659 case RTE_FLOW_ITEM_TYPE_VOID:
1661 case RTE_FLOW_ITEM_TYPE_ETH:
1662 flow_verbs_translate_item_eth(dev_flow, items,
1664 subpriority = MLX5_PRIORITY_MAP_L2;
1665 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1666 MLX5_FLOW_LAYER_OUTER_L2;
1668 case RTE_FLOW_ITEM_TYPE_VLAN:
1669 flow_verbs_translate_item_vlan(dev_flow, items,
1671 subpriority = MLX5_PRIORITY_MAP_L2;
1672 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1673 MLX5_FLOW_LAYER_INNER_VLAN) :
1674 (MLX5_FLOW_LAYER_OUTER_L2 |
1675 MLX5_FLOW_LAYER_OUTER_VLAN);
1677 case RTE_FLOW_ITEM_TYPE_IPV4:
1678 flow_verbs_translate_item_ipv4(dev_flow, items,
1680 subpriority = MLX5_PRIORITY_MAP_L3;
1681 dev_flow->hash_fields |=
1682 mlx5_flow_hashfields_adjust
1684 MLX5_IPV4_LAYER_TYPES,
1685 MLX5_IPV4_IBV_RX_HASH);
1686 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1687 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1689 case RTE_FLOW_ITEM_TYPE_IPV6:
1690 flow_verbs_translate_item_ipv6(dev_flow, items,
1692 subpriority = MLX5_PRIORITY_MAP_L3;
1693 dev_flow->hash_fields |=
1694 mlx5_flow_hashfields_adjust
1696 MLX5_IPV6_LAYER_TYPES,
1697 MLX5_IPV6_IBV_RX_HASH);
1698 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1699 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1701 case RTE_FLOW_ITEM_TYPE_TCP:
1702 flow_verbs_translate_item_tcp(dev_flow, items,
1704 subpriority = MLX5_PRIORITY_MAP_L4;
1705 dev_flow->hash_fields |=
1706 mlx5_flow_hashfields_adjust
1707 (rss_desc, tunnel, ETH_RSS_TCP,
1708 (IBV_RX_HASH_SRC_PORT_TCP |
1709 IBV_RX_HASH_DST_PORT_TCP));
1710 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1711 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1713 case RTE_FLOW_ITEM_TYPE_UDP:
1714 flow_verbs_translate_item_udp(dev_flow, items,
1716 subpriority = MLX5_PRIORITY_MAP_L4;
1717 dev_flow->hash_fields |=
1718 mlx5_flow_hashfields_adjust
1719 (rss_desc, tunnel, ETH_RSS_UDP,
1720 (IBV_RX_HASH_SRC_PORT_UDP |
1721 IBV_RX_HASH_DST_PORT_UDP));
1722 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1723 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1725 case RTE_FLOW_ITEM_TYPE_VXLAN:
1726 flow_verbs_translate_item_vxlan(dev_flow, items,
1728 subpriority = MLX5_PRIORITY_MAP_L2;
1729 item_flags |= MLX5_FLOW_LAYER_VXLAN;
1731 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1732 flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1734 subpriority = MLX5_PRIORITY_MAP_L2;
1735 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1737 case RTE_FLOW_ITEM_TYPE_GRE:
1738 flow_verbs_translate_item_gre(dev_flow, items,
1740 subpriority = MLX5_PRIORITY_MAP_L2;
1741 item_flags |= MLX5_FLOW_LAYER_GRE;
1743 case RTE_FLOW_ITEM_TYPE_MPLS:
1744 flow_verbs_translate_item_mpls(dev_flow, items,
1746 subpriority = MLX5_PRIORITY_MAP_L2;
1747 item_flags |= MLX5_FLOW_LAYER_MPLS;
1750 return rte_flow_error_set(error, ENOTSUP,
1751 RTE_FLOW_ERROR_TYPE_ITEM,
1753 "item not supported");
1756 dev_flow->handle->layers = item_flags;
1757 /* Other members of attr will be ignored. */
1758 dev_flow->verbs.attr.priority =
1759 mlx5_flow_adjust_priority(dev, priority, subpriority);
1760 dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1765 * Remove the flow from the NIC but keeps it in memory.
1768 * Pointer to the Ethernet device structure.
1769 * @param[in, out] flow
1770 * Pointer to flow structure.
1773 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1775 struct mlx5_priv *priv = dev->data->dev_private;
1776 struct mlx5_flow_handle *handle;
1777 uint32_t handle_idx;
1781 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1782 handle_idx, handle, next) {
1783 if (handle->drv_flow) {
1784 claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1785 handle->drv_flow = NULL;
1787 /* hrxq is union, don't touch it only the flag is set. */
1788 if (handle->rix_hrxq) {
1789 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1790 mlx5_hrxq_drop_release(dev);
1791 handle->rix_hrxq = 0;
1792 } else if (handle->fate_action ==
1793 MLX5_FLOW_FATE_QUEUE) {
1794 mlx5_hrxq_release(dev, handle->rix_hrxq);
1795 handle->rix_hrxq = 0;
1798 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1799 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1804 * Remove the flow from the NIC and the memory.
1807 * Pointer to the Ethernet device structure.
1808 * @param[in, out] flow
1809 * Pointer to flow structure.
1812 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1814 struct mlx5_priv *priv = dev->data->dev_private;
1815 struct mlx5_flow_handle *handle;
1819 flow_verbs_remove(dev, flow);
1820 while (flow->dev_handles) {
1821 uint32_t tmp_idx = flow->dev_handles;
1823 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1827 flow->dev_handles = handle->next.next;
1828 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1831 if (flow->counter) {
1832 flow_verbs_counter_release(dev, flow->counter);
1838 * Apply the flow to the NIC.
1841 * Pointer to the Ethernet device structure.
1842 * @param[in, out] flow
1843 * Pointer to flow structure.
1845 * Pointer to error structure.
1848 * 0 on success, a negative errno value otherwise and rte_errno is set.
1851 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1852 struct rte_flow_error *error)
1854 struct mlx5_priv *priv = dev->data->dev_private;
1855 struct mlx5_flow_handle *handle;
1856 struct mlx5_flow *dev_flow;
1857 struct mlx5_hrxq *hrxq;
1858 uint32_t dev_handles;
1862 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
1863 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1864 handle = dev_flow->handle;
1865 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1866 hrxq = mlx5_hrxq_drop_new(dev);
1870 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1871 "cannot get drop hash queue");
1876 struct mlx5_flow_rss_desc *rss_desc =
1877 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
1878 [!!priv->flow_nested_idx];
1880 MLX5_ASSERT(rss_desc->queue_num);
1881 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
1882 MLX5_RSS_HASH_KEY_LEN,
1883 dev_flow->hash_fields,
1885 rss_desc->queue_num);
1887 hrxq_idx = mlx5_hrxq_new(dev, rss_desc->key,
1888 MLX5_RSS_HASH_KEY_LEN,
1889 dev_flow->hash_fields,
1891 rss_desc->queue_num,
1893 MLX5_FLOW_LAYER_TUNNEL));
1894 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1899 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1900 "cannot get hash queue");
1903 handle->rix_hrxq = hrxq_idx;
1906 handle->drv_flow = mlx5_glue->create_flow
1907 (hrxq->qp, &dev_flow->verbs.attr);
1908 if (!handle->drv_flow) {
1909 rte_flow_error_set(error, errno,
1910 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1912 "hardware refuses to create flow");
1915 if (priv->vmwa_context &&
1916 handle->vf_vlan.tag && !handle->vf_vlan.created) {
1918 * The rule contains the VLAN pattern.
1919 * For VF we are going to create VLAN
1920 * interface to make hypervisor set correct
1921 * e-Switch vport context.
1923 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
1928 err = rte_errno; /* Save rte_errno before cleanup. */
1929 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1930 dev_handles, handle, next) {
1931 /* hrxq is union, don't touch it only the flag is set. */
1932 if (handle->rix_hrxq) {
1933 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1934 mlx5_hrxq_drop_release(dev);
1935 handle->rix_hrxq = 0;
1936 } else if (handle->fate_action ==
1937 MLX5_FLOW_FATE_QUEUE) {
1938 mlx5_hrxq_release(dev, handle->rix_hrxq);
1939 handle->rix_hrxq = 0;
1942 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1943 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1945 rte_errno = err; /* Restore rte_errno. */
1952 * @see rte_flow_query()
1956 flow_verbs_query(struct rte_eth_dev *dev,
1957 struct rte_flow *flow,
1958 const struct rte_flow_action *actions,
1960 struct rte_flow_error *error)
1964 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1965 switch (actions->type) {
1966 case RTE_FLOW_ACTION_TYPE_VOID:
1968 case RTE_FLOW_ACTION_TYPE_COUNT:
1969 ret = flow_verbs_counter_query(dev, flow, data, error);
1972 return rte_flow_error_set(error, ENOTSUP,
1973 RTE_FLOW_ERROR_TYPE_ACTION,
1975 "action not supported");
1981 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1982 .validate = flow_verbs_validate,
1983 .prepare = flow_verbs_prepare,
1984 .translate = flow_verbs_translate,
1985 .apply = flow_verbs_apply,
1986 .remove = flow_verbs_remove,
1987 .destroy = flow_verbs_destroy,
1988 .query = flow_verbs_query,