1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
5 #include <netinet/in.h>
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
29 #include <mlx5_glue.h>
32 #include "mlx5_defs.h"
34 #include "mlx5_flow.h"
35 #include "mlx5_rxtx.h"
37 #define VERBS_SPEC_INNER(item_flags) \
38 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
41 * Get Verbs flow counter by index.
44 * Pointer to the Ethernet device structure.
46 * mlx5 flow counter index in the container.
48 * mlx5 flow counter pool in the container,
51 * A pointer to the counter, NULL otherwise.
53 static struct mlx5_flow_counter *
54 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
56 struct mlx5_flow_counter_pool **ppool)
58 struct mlx5_priv *priv = dev->data->dev_private;
59 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
60 struct mlx5_flow_counter_pool *pool;
63 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
67 return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL];
71 * Create Verbs flow counter with Verbs library.
74 * Pointer to the Ethernet device structure.
75 * @param[in, out] counter
76 * mlx5 flow counter object, contains the counter id,
77 * handle of created Verbs flow counter is returned
78 * in cs field (if counters are supported).
81 * 0 On success else a negative errno value is returned
82 * and rte_errno is set.
85 flow_verbs_counter_create(struct rte_eth_dev *dev,
86 struct mlx5_flow_counter_ext *counter)
88 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
89 struct mlx5_priv *priv = dev->data->dev_private;
90 struct ibv_context *ctx = priv->sh->ctx;
91 struct ibv_counter_set_init_attr init = {
92 .counter_set_id = counter->id};
94 counter->cs = mlx5_glue->create_counter_set(ctx, &init);
100 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
101 struct mlx5_priv *priv = dev->data->dev_private;
102 struct ibv_context *ctx = priv->sh->ctx;
103 struct ibv_counters_init_attr init = {0};
104 struct ibv_counter_attach_attr attach;
107 memset(&attach, 0, sizeof(attach));
108 counter->cs = mlx5_glue->create_counters(ctx, &init);
113 attach.counter_desc = IBV_COUNTER_PACKETS;
115 ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
117 attach.counter_desc = IBV_COUNTER_BYTES;
119 ret = mlx5_glue->attach_counters
120 (counter->cs, &attach, NULL);
123 claim_zero(mlx5_glue->destroy_counters(counter->cs));
138 * Get a flow counter.
141 * Pointer to the Ethernet device structure.
143 * Indicate if this counter is shared with other flows.
145 * Counter identifier.
148 * Index to the counter, 0 otherwise and rte_errno is set.
151 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
153 struct mlx5_priv *priv = dev->data->dev_private;
154 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
155 struct mlx5_flow_counter_pool *pool = NULL;
156 struct mlx5_flow_counter_ext *cnt_ext = NULL;
157 struct mlx5_flow_counter *cnt = NULL;
158 uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
164 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
165 pool = cont->pools[pool_idx];
166 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
167 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
168 if (cnt_ext->shared && cnt_ext->id == id) {
170 return MLX5_MAKE_CNT_IDX(pool_idx, i);
175 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
176 pool = cont->pools[pool_idx];
179 cnt = TAILQ_FIRST(&pool->counters);
184 struct mlx5_flow_counter_pool **pools;
187 if (n_valid == cont->n) {
188 /* Resize the container pool array. */
189 size = sizeof(struct mlx5_flow_counter_pool *) *
190 (n_valid + MLX5_CNT_CONTAINER_RESIZE);
191 pools = rte_zmalloc(__func__, size, 0);
195 memcpy(pools, cont->pools,
196 sizeof(struct mlx5_flow_counter_pool *) *
198 rte_free(cont->pools);
201 cont->n += MLX5_CNT_CONTAINER_RESIZE;
203 /* Allocate memory for new pool*/
204 size = sizeof(*pool) + sizeof(*cnt_ext) *
205 MLX5_COUNTERS_PER_POOL;
206 pool = rte_calloc(__func__, 1, size, 0);
209 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
210 cnt = &pool->counters_raw[i];
211 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
213 cnt = &pool->counters_raw[0];
214 cont->pools[n_valid] = pool;
216 rte_atomic16_add(&cont->n_valid, 1);
217 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
219 i = cnt - pool->counters_raw;
220 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
222 cnt_ext->shared = shared;
223 cnt_ext->ref_cnt = 1;
226 /* Create counter with Verbs. */
227 ret = flow_verbs_counter_create(dev, cnt_ext);
229 TAILQ_REMOVE(&pool->counters, cnt, next);
230 return MLX5_MAKE_CNT_IDX(pool_idx, i);
232 /* Some error occurred in Verbs library. */
238 * Release a flow counter.
241 * Pointer to the Ethernet device structure.
243 * Index to the counter handler.
246 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
248 struct mlx5_flow_counter_pool *pool;
249 struct mlx5_flow_counter *cnt;
250 struct mlx5_flow_counter_ext *cnt_ext;
252 cnt = flow_verbs_counter_get_by_idx(dev, counter,
254 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
255 if (--cnt_ext->ref_cnt == 0) {
256 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
257 claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
259 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
260 claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs));
263 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
268 * Query a flow counter via Verbs library call.
270 * @see rte_flow_query()
274 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
275 struct rte_flow *flow, void *data,
276 struct rte_flow_error *error)
278 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
279 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
281 struct mlx5_flow_counter_pool *pool;
282 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
283 (dev, flow->counter, &pool);
284 struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT
286 struct rte_flow_query_count *qc = data;
287 uint64_t counters[2] = {0, 0};
288 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
289 struct ibv_query_counter_set_attr query_cs_attr = {
291 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
293 struct ibv_counter_set_data query_out = {
295 .outlen = 2 * sizeof(uint64_t),
297 int err = mlx5_glue->query_counter_set(&query_cs_attr,
299 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
300 int err = mlx5_glue->query_counters
301 (cnt_ext->cs, counters,
303 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
306 return rte_flow_error_set
308 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
310 "cannot read counter");
313 qc->hits = counters[0] - cnt->hits;
314 qc->bytes = counters[1] - cnt->bytes;
316 cnt->hits = counters[0];
317 cnt->bytes = counters[1];
321 return rte_flow_error_set(error, EINVAL,
322 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
324 "flow does not have counter");
328 return rte_flow_error_set(error, ENOTSUP,
329 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
331 "counters are not available");
336 * Add a verbs item specification into @p verbs.
339 * Pointer to verbs structure.
341 * Create specification.
343 * Size in bytes of the specification to copy.
346 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
347 void *src, unsigned int size)
353 MLX5_ASSERT(verbs->specs);
354 dst = (void *)(verbs->specs + verbs->size);
355 memcpy(dst, src, size);
356 ++verbs->attr.num_of_specs;
361 * Convert the @p item into a Verbs specification. This function assumes that
362 * the input is valid and that there is space to insert the requested item
365 * @param[in, out] dev_flow
366 * Pointer to dev_flow structure.
368 * Item specification.
369 * @param[in] item_flags
373 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
374 const struct rte_flow_item *item,
377 const struct rte_flow_item_eth *spec = item->spec;
378 const struct rte_flow_item_eth *mask = item->mask;
379 const unsigned int size = sizeof(struct ibv_flow_spec_eth);
380 struct ibv_flow_spec_eth eth = {
381 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
386 mask = &rte_flow_item_eth_mask;
390 memcpy(ð.val.dst_mac, spec->dst.addr_bytes,
392 memcpy(ð.val.src_mac, spec->src.addr_bytes,
394 eth.val.ether_type = spec->type;
395 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes,
397 memcpy(ð.mask.src_mac, mask->src.addr_bytes,
399 eth.mask.ether_type = mask->type;
400 /* Remove unwanted bits from values. */
401 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
402 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
403 eth.val.src_mac[i] &= eth.mask.src_mac[i];
405 eth.val.ether_type &= eth.mask.ether_type;
407 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
411 * Update the VLAN tag in the Verbs Ethernet specification.
412 * This function assumes that the input is valid and there is space to add
413 * the requested item.
415 * @param[in, out] attr
416 * Pointer to Verbs attributes structure.
418 * Verbs structure containing the VLAN information to copy.
421 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
422 struct ibv_flow_spec_eth *eth)
425 const enum ibv_flow_spec_type search = eth->type;
426 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
427 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
429 for (i = 0; i != attr->num_of_specs; ++i) {
430 if (hdr->type == search) {
431 struct ibv_flow_spec_eth *e =
432 (struct ibv_flow_spec_eth *)hdr;
434 e->val.vlan_tag = eth->val.vlan_tag;
435 e->mask.vlan_tag = eth->mask.vlan_tag;
436 e->val.ether_type = eth->val.ether_type;
437 e->mask.ether_type = eth->mask.ether_type;
440 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
445 * Convert the @p item into a Verbs specification. This function assumes that
446 * the input is valid and that there is space to insert the requested item
449 * @param[in, out] dev_flow
450 * Pointer to dev_flow structure.
452 * Item specification.
453 * @param[in] item_flags
457 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
458 const struct rte_flow_item *item,
461 const struct rte_flow_item_vlan *spec = item->spec;
462 const struct rte_flow_item_vlan *mask = item->mask;
463 unsigned int size = sizeof(struct ibv_flow_spec_eth);
464 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
465 struct ibv_flow_spec_eth eth = {
466 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
469 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
470 MLX5_FLOW_LAYER_OUTER_L2;
473 mask = &rte_flow_item_vlan_mask;
475 eth.val.vlan_tag = spec->tci;
476 eth.mask.vlan_tag = mask->tci;
477 eth.val.vlan_tag &= eth.mask.vlan_tag;
478 eth.val.ether_type = spec->inner_type;
479 eth.mask.ether_type = mask->inner_type;
480 eth.val.ether_type &= eth.mask.ether_type;
482 if (!(item_flags & l2m))
483 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
485 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð);
487 dev_flow->handle->vf_vlan.tag =
488 rte_be_to_cpu_16(spec->tci) & 0x0fff;
492 * Convert the @p item into a Verbs specification. This function assumes that
493 * the input is valid and that there is space to insert the requested item
496 * @param[in, out] dev_flow
497 * Pointer to dev_flow structure.
499 * Item specification.
500 * @param[in] item_flags
504 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
505 const struct rte_flow_item *item,
508 const struct rte_flow_item_ipv4 *spec = item->spec;
509 const struct rte_flow_item_ipv4 *mask = item->mask;
510 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
511 struct ibv_flow_spec_ipv4_ext ipv4 = {
512 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
517 mask = &rte_flow_item_ipv4_mask;
519 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
520 .src_ip = spec->hdr.src_addr,
521 .dst_ip = spec->hdr.dst_addr,
522 .proto = spec->hdr.next_proto_id,
523 .tos = spec->hdr.type_of_service,
525 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
526 .src_ip = mask->hdr.src_addr,
527 .dst_ip = mask->hdr.dst_addr,
528 .proto = mask->hdr.next_proto_id,
529 .tos = mask->hdr.type_of_service,
531 /* Remove unwanted bits from values. */
532 ipv4.val.src_ip &= ipv4.mask.src_ip;
533 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
534 ipv4.val.proto &= ipv4.mask.proto;
535 ipv4.val.tos &= ipv4.mask.tos;
537 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
541 * Convert the @p item into a Verbs specification. This function assumes that
542 * the input is valid and that there is space to insert the requested item
545 * @param[in, out] dev_flow
546 * Pointer to dev_flow structure.
548 * Item specification.
549 * @param[in] item_flags
553 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
554 const struct rte_flow_item *item,
557 const struct rte_flow_item_ipv6 *spec = item->spec;
558 const struct rte_flow_item_ipv6 *mask = item->mask;
559 unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
560 struct ibv_flow_spec_ipv6 ipv6 = {
561 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
566 mask = &rte_flow_item_ipv6_mask;
569 uint32_t vtc_flow_val;
570 uint32_t vtc_flow_mask;
572 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
573 RTE_DIM(ipv6.val.src_ip));
574 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
575 RTE_DIM(ipv6.val.dst_ip));
576 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
577 RTE_DIM(ipv6.mask.src_ip));
578 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
579 RTE_DIM(ipv6.mask.dst_ip));
580 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
581 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
582 ipv6.val.flow_label =
583 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
584 RTE_IPV6_HDR_FL_SHIFT);
585 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
586 RTE_IPV6_HDR_TC_SHIFT;
587 ipv6.val.next_hdr = spec->hdr.proto;
588 ipv6.mask.flow_label =
589 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
590 RTE_IPV6_HDR_FL_SHIFT);
591 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
592 RTE_IPV6_HDR_TC_SHIFT;
593 ipv6.mask.next_hdr = mask->hdr.proto;
594 /* Remove unwanted bits from values. */
595 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
596 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
597 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
599 ipv6.val.flow_label &= ipv6.mask.flow_label;
600 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
601 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
603 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
607 * Convert the @p item into a Verbs specification. This function assumes that
608 * the input is valid and that there is space to insert the requested item
611 * @param[in, out] dev_flow
612 * Pointer to dev_flow structure.
614 * Item specification.
615 * @param[in] item_flags
619 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
620 const struct rte_flow_item *item,
621 uint64_t item_flags __rte_unused)
623 const struct rte_flow_item_tcp *spec = item->spec;
624 const struct rte_flow_item_tcp *mask = item->mask;
625 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
626 struct ibv_flow_spec_tcp_udp tcp = {
627 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
632 mask = &rte_flow_item_tcp_mask;
634 tcp.val.dst_port = spec->hdr.dst_port;
635 tcp.val.src_port = spec->hdr.src_port;
636 tcp.mask.dst_port = mask->hdr.dst_port;
637 tcp.mask.src_port = mask->hdr.src_port;
638 /* Remove unwanted bits from values. */
639 tcp.val.src_port &= tcp.mask.src_port;
640 tcp.val.dst_port &= tcp.mask.dst_port;
642 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
646 * Convert the @p item into a Verbs specification. This function assumes that
647 * the input is valid and that there is space to insert the requested item
650 * @param[in, out] dev_flow
651 * Pointer to dev_flow structure.
653 * Item specification.
654 * @param[in] item_flags
658 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
659 const struct rte_flow_item *item,
660 uint64_t item_flags __rte_unused)
662 const struct rte_flow_item_udp *spec = item->spec;
663 const struct rte_flow_item_udp *mask = item->mask;
664 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
665 struct ibv_flow_spec_tcp_udp udp = {
666 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
671 mask = &rte_flow_item_udp_mask;
673 udp.val.dst_port = spec->hdr.dst_port;
674 udp.val.src_port = spec->hdr.src_port;
675 udp.mask.dst_port = mask->hdr.dst_port;
676 udp.mask.src_port = mask->hdr.src_port;
677 /* Remove unwanted bits from values. */
678 udp.val.src_port &= udp.mask.src_port;
679 udp.val.dst_port &= udp.mask.dst_port;
681 flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
685 * Convert the @p item into a Verbs specification. This function assumes that
686 * the input is valid and that there is space to insert the requested item
689 * @param[in, out] dev_flow
690 * Pointer to dev_flow structure.
692 * Item specification.
693 * @param[in] item_flags
697 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
698 const struct rte_flow_item *item,
699 uint64_t item_flags __rte_unused)
701 const struct rte_flow_item_vxlan *spec = item->spec;
702 const struct rte_flow_item_vxlan *mask = item->mask;
703 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
704 struct ibv_flow_spec_tunnel vxlan = {
705 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
711 } id = { .vlan_id = 0, };
714 mask = &rte_flow_item_vxlan_mask;
716 memcpy(&id.vni[1], spec->vni, 3);
717 vxlan.val.tunnel_id = id.vlan_id;
718 memcpy(&id.vni[1], mask->vni, 3);
719 vxlan.mask.tunnel_id = id.vlan_id;
720 /* Remove unwanted bits from values. */
721 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
723 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
727 * Convert the @p item into a Verbs specification. This function assumes that
728 * the input is valid and that there is space to insert the requested item
731 * @param[in, out] dev_flow
732 * Pointer to dev_flow structure.
734 * Item specification.
735 * @param[in] item_flags
739 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
740 const struct rte_flow_item *item,
741 uint64_t item_flags __rte_unused)
743 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
744 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
745 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
746 struct ibv_flow_spec_tunnel vxlan_gpe = {
747 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
753 } id = { .vlan_id = 0, };
756 mask = &rte_flow_item_vxlan_gpe_mask;
758 memcpy(&id.vni[1], spec->vni, 3);
759 vxlan_gpe.val.tunnel_id = id.vlan_id;
760 memcpy(&id.vni[1], mask->vni, 3);
761 vxlan_gpe.mask.tunnel_id = id.vlan_id;
762 /* Remove unwanted bits from values. */
763 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
765 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
769 * Update the protocol in Verbs IPv4/IPv6 spec.
771 * @param[in, out] attr
772 * Pointer to Verbs attributes structure.
774 * Specification type to search in order to update the IP protocol.
775 * @param[in] protocol
776 * Protocol value to set if none is present in the specification.
779 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
780 enum ibv_flow_spec_type search,
784 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
785 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
789 for (i = 0; i != attr->num_of_specs; ++i) {
790 if (hdr->type == search) {
792 struct ibv_flow_spec_ipv4_ext *ipv4;
793 struct ibv_flow_spec_ipv6 *ipv6;
797 case IBV_FLOW_SPEC_IPV4_EXT:
798 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
799 if (!ip.ipv4->val.proto) {
800 ip.ipv4->val.proto = protocol;
801 ip.ipv4->mask.proto = 0xff;
804 case IBV_FLOW_SPEC_IPV6:
805 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
806 if (!ip.ipv6->val.next_hdr) {
807 ip.ipv6->val.next_hdr = protocol;
808 ip.ipv6->mask.next_hdr = 0xff;
816 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
821 * Convert the @p item into a Verbs specification. This function assumes that
822 * the input is valid and that there is space to insert the requested item
825 * @param[in, out] dev_flow
826 * Pointer to dev_flow structure.
828 * Item specification.
829 * @param[in] item_flags
833 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
834 const struct rte_flow_item *item __rte_unused,
837 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
838 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
839 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
840 struct ibv_flow_spec_tunnel tunnel = {
841 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
845 const struct rte_flow_item_gre *spec = item->spec;
846 const struct rte_flow_item_gre *mask = item->mask;
847 unsigned int size = sizeof(struct ibv_flow_spec_gre);
848 struct ibv_flow_spec_gre tunnel = {
849 .type = IBV_FLOW_SPEC_GRE,
854 mask = &rte_flow_item_gre_mask;
856 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
857 tunnel.val.protocol = spec->protocol;
858 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
859 tunnel.mask.protocol = mask->protocol;
860 /* Remove unwanted bits from values. */
861 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
862 tunnel.val.protocol &= tunnel.mask.protocol;
863 tunnel.val.key &= tunnel.mask.key;
866 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
867 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
868 IBV_FLOW_SPEC_IPV4_EXT,
871 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
874 flow_verbs_spec_add(verbs, &tunnel, size);
878 * Convert the @p action into a Verbs specification. This function assumes that
879 * the input is valid and that there is space to insert the requested action
880 * into the flow. This function also return the action that was added.
882 * @param[in, out] dev_flow
883 * Pointer to dev_flow structure.
885 * Item specification.
886 * @param[in] item_flags
890 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
891 const struct rte_flow_item *item __rte_unused,
892 uint64_t item_flags __rte_unused)
894 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
895 const struct rte_flow_item_mpls *spec = item->spec;
896 const struct rte_flow_item_mpls *mask = item->mask;
897 unsigned int size = sizeof(struct ibv_flow_spec_mpls);
898 struct ibv_flow_spec_mpls mpls = {
899 .type = IBV_FLOW_SPEC_MPLS,
904 mask = &rte_flow_item_mpls_mask;
906 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
907 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
908 /* Remove unwanted bits from values. */
909 mpls.val.label &= mpls.mask.label;
911 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
916 * Convert the @p action into a Verbs specification. This function assumes that
917 * the input is valid and that there is space to insert the requested action
920 * @param[in] dev_flow
921 * Pointer to mlx5_flow.
923 * Action configuration.
926 flow_verbs_translate_action_drop
927 (struct mlx5_flow *dev_flow,
928 const struct rte_flow_action *action __rte_unused)
930 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
931 struct ibv_flow_spec_action_drop drop = {
932 .type = IBV_FLOW_SPEC_ACTION_DROP,
936 flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
940 * Convert the @p action into a Verbs specification. This function assumes that
941 * the input is valid and that there is space to insert the requested action
944 * @param[in] dev_flow
945 * Pointer to mlx5_flow.
947 * Action configuration.
950 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
951 const struct rte_flow_action *action)
953 const struct rte_flow_action_queue *queue = action->conf;
954 struct rte_flow *flow = dev_flow->flow;
957 (*flow->rss.queue)[0] = queue->index;
958 flow->rss.queue_num = 1;
962 * Convert the @p action into a Verbs specification. This function assumes that
963 * the input is valid and that there is space to insert the requested action
967 * Action configuration.
968 * @param[in, out] action_flags
969 * Pointer to the detected actions.
970 * @param[in] dev_flow
971 * Pointer to mlx5_flow.
974 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
975 const struct rte_flow_action *action)
977 const struct rte_flow_action_rss *rss = action->conf;
978 const uint8_t *rss_key;
979 struct rte_flow *flow = dev_flow->flow;
982 memcpy((*flow->rss.queue), rss->queue,
983 rss->queue_num * sizeof(uint16_t));
984 flow->rss.queue_num = rss->queue_num;
985 /* NULL RSS key indicates default RSS key. */
986 rss_key = !rss->key ? rss_hash_default_key : rss->key;
987 memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
989 * rss->level and rss.types should be set in advance when expanding
995 * Convert the @p action into a Verbs specification. This function assumes that
996 * the input is valid and that there is space to insert the requested action
999 * @param[in] dev_flow
1000 * Pointer to mlx5_flow.
1002 * Action configuration.
1005 flow_verbs_translate_action_flag
1006 (struct mlx5_flow *dev_flow,
1007 const struct rte_flow_action *action __rte_unused)
1009 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1010 struct ibv_flow_spec_action_tag tag = {
1011 .type = IBV_FLOW_SPEC_ACTION_TAG,
1013 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1016 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1020 * Convert the @p action into a Verbs specification. This function assumes that
1021 * the input is valid and that there is space to insert the requested action
1024 * @param[in] dev_flow
1025 * Pointer to mlx5_flow.
1027 * Action configuration.
1030 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1031 const struct rte_flow_action *action)
1033 const struct rte_flow_action_mark *mark = action->conf;
1034 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1035 struct ibv_flow_spec_action_tag tag = {
1036 .type = IBV_FLOW_SPEC_ACTION_TAG,
1038 .tag_id = mlx5_flow_mark_set(mark->id),
1041 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1045 * Convert the @p action into a Verbs specification. This function assumes that
1046 * the input is valid and that there is space to insert the requested action
1050 * Pointer to the Ethernet device structure.
1052 * Action configuration.
1053 * @param[in] dev_flow
1054 * Pointer to mlx5_flow.
1056 * Pointer to error structure.
1059 * 0 On success else a negative errno value is returned and rte_errno is set.
1062 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1063 const struct rte_flow_action *action,
1064 struct rte_eth_dev *dev,
1065 struct rte_flow_error *error)
1067 const struct rte_flow_action_count *count = action->conf;
1068 struct rte_flow *flow = dev_flow->flow;
1069 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1070 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1071 struct mlx5_flow_counter_pool *pool;
1072 struct mlx5_flow_counter *cnt = NULL;
1073 struct mlx5_flow_counter_ext *cnt_ext;
1074 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1075 struct ibv_flow_spec_counter_action counter = {
1076 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1081 if (!flow->counter) {
1082 flow->counter = flow_verbs_counter_new(dev, count->shared,
1085 return rte_flow_error_set(error, rte_errno,
1086 RTE_FLOW_ERROR_TYPE_ACTION,
1088 "cannot get counter"
1091 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1092 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1093 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1094 counter.counter_set_handle = cnt_ext->cs->handle;
1095 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1096 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1097 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1098 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1099 counter.counters = cnt_ext->cs;
1100 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1106 * Internal validation function. For validating both actions and items.
1109 * Pointer to the Ethernet device structure.
1111 * Pointer to the flow attributes.
1113 * Pointer to the list of items.
1114 * @param[in] actions
1115 * Pointer to the list of actions.
1116 * @param[in] external
1117 * This flow rule is created by request external to PMD.
1119 * Pointer to the error structure.
1122 * 0 on success, a negative errno value otherwise and rte_errno is set.
1125 flow_verbs_validate(struct rte_eth_dev *dev,
1126 const struct rte_flow_attr *attr,
1127 const struct rte_flow_item items[],
1128 const struct rte_flow_action actions[],
1129 bool external __rte_unused,
1130 struct rte_flow_error *error)
1133 uint64_t action_flags = 0;
1134 uint64_t item_flags = 0;
1135 uint64_t last_item = 0;
1136 uint8_t next_protocol = 0xff;
1137 uint16_t ether_type = 0;
1141 ret = mlx5_flow_validate_attributes(dev, attr, error);
1144 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1145 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1148 switch (items->type) {
1149 case RTE_FLOW_ITEM_TYPE_VOID:
1151 case RTE_FLOW_ITEM_TYPE_ETH:
1152 ret = mlx5_flow_validate_item_eth(items, item_flags,
1156 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1157 MLX5_FLOW_LAYER_OUTER_L2;
1158 if (items->mask != NULL && items->spec != NULL) {
1160 ((const struct rte_flow_item_eth *)
1163 ((const struct rte_flow_item_eth *)
1165 ether_type = rte_be_to_cpu_16(ether_type);
1170 case RTE_FLOW_ITEM_TYPE_VLAN:
1171 ret = mlx5_flow_validate_item_vlan(items, item_flags,
1175 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1176 MLX5_FLOW_LAYER_INNER_VLAN) :
1177 (MLX5_FLOW_LAYER_OUTER_L2 |
1178 MLX5_FLOW_LAYER_OUTER_VLAN);
1179 if (items->mask != NULL && items->spec != NULL) {
1181 ((const struct rte_flow_item_vlan *)
1182 items->spec)->inner_type;
1184 ((const struct rte_flow_item_vlan *)
1185 items->mask)->inner_type;
1186 ether_type = rte_be_to_cpu_16(ether_type);
1191 case RTE_FLOW_ITEM_TYPE_IPV4:
1192 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1198 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1199 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1200 if (items->mask != NULL &&
1201 ((const struct rte_flow_item_ipv4 *)
1202 items->mask)->hdr.next_proto_id) {
1204 ((const struct rte_flow_item_ipv4 *)
1205 (items->spec))->hdr.next_proto_id;
1207 ((const struct rte_flow_item_ipv4 *)
1208 (items->mask))->hdr.next_proto_id;
1210 /* Reset for inner layer. */
1211 next_protocol = 0xff;
1214 case RTE_FLOW_ITEM_TYPE_IPV6:
1215 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1221 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1222 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1223 if (items->mask != NULL &&
1224 ((const struct rte_flow_item_ipv6 *)
1225 items->mask)->hdr.proto) {
1227 ((const struct rte_flow_item_ipv6 *)
1228 items->spec)->hdr.proto;
1230 ((const struct rte_flow_item_ipv6 *)
1231 items->mask)->hdr.proto;
1233 /* Reset for inner layer. */
1234 next_protocol = 0xff;
1237 case RTE_FLOW_ITEM_TYPE_UDP:
1238 ret = mlx5_flow_validate_item_udp(items, item_flags,
1243 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1244 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1246 case RTE_FLOW_ITEM_TYPE_TCP:
1247 ret = mlx5_flow_validate_item_tcp
1250 &rte_flow_item_tcp_mask,
1254 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1255 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1257 case RTE_FLOW_ITEM_TYPE_VXLAN:
1258 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1262 last_item = MLX5_FLOW_LAYER_VXLAN;
1264 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1265 ret = mlx5_flow_validate_item_vxlan_gpe(items,
1270 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1272 case RTE_FLOW_ITEM_TYPE_GRE:
1273 ret = mlx5_flow_validate_item_gre(items, item_flags,
1274 next_protocol, error);
1277 last_item = MLX5_FLOW_LAYER_GRE;
1279 case RTE_FLOW_ITEM_TYPE_MPLS:
1280 ret = mlx5_flow_validate_item_mpls(dev, items,
1285 last_item = MLX5_FLOW_LAYER_MPLS;
1288 return rte_flow_error_set(error, ENOTSUP,
1289 RTE_FLOW_ERROR_TYPE_ITEM,
1290 NULL, "item not supported");
1292 item_flags |= last_item;
1294 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1295 switch (actions->type) {
1296 case RTE_FLOW_ACTION_TYPE_VOID:
1298 case RTE_FLOW_ACTION_TYPE_FLAG:
1299 ret = mlx5_flow_validate_action_flag(action_flags,
1304 action_flags |= MLX5_FLOW_ACTION_FLAG;
1306 case RTE_FLOW_ACTION_TYPE_MARK:
1307 ret = mlx5_flow_validate_action_mark(actions,
1313 action_flags |= MLX5_FLOW_ACTION_MARK;
1315 case RTE_FLOW_ACTION_TYPE_DROP:
1316 ret = mlx5_flow_validate_action_drop(action_flags,
1321 action_flags |= MLX5_FLOW_ACTION_DROP;
1323 case RTE_FLOW_ACTION_TYPE_QUEUE:
1324 ret = mlx5_flow_validate_action_queue(actions,
1330 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1332 case RTE_FLOW_ACTION_TYPE_RSS:
1333 ret = mlx5_flow_validate_action_rss(actions,
1339 action_flags |= MLX5_FLOW_ACTION_RSS;
1341 case RTE_FLOW_ACTION_TYPE_COUNT:
1342 ret = mlx5_flow_validate_action_count(dev, attr, error);
1345 action_flags |= MLX5_FLOW_ACTION_COUNT;
1348 return rte_flow_error_set(error, ENOTSUP,
1349 RTE_FLOW_ERROR_TYPE_ACTION,
1351 "action not supported");
1355 * Validate the drop action mutual exclusion with other actions.
1356 * Drop action is mutually-exclusive with any other action, except for
1359 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1360 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1361 return rte_flow_error_set(error, EINVAL,
1362 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1363 "Drop action is mutually-exclusive "
1364 "with any other action, except for "
1366 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1367 return rte_flow_error_set(error, EINVAL,
1368 RTE_FLOW_ERROR_TYPE_ACTION, actions,
1369 "no fate action is found");
1374 * Calculate the required bytes that are needed for the action part of the verbs
1377 * @param[in] actions
1378 * Pointer to the list of actions.
1381 * The size of the memory needed for all actions.
1384 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1388 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1389 switch (actions->type) {
1390 case RTE_FLOW_ACTION_TYPE_VOID:
1392 case RTE_FLOW_ACTION_TYPE_FLAG:
1393 size += sizeof(struct ibv_flow_spec_action_tag);
1395 case RTE_FLOW_ACTION_TYPE_MARK:
1396 size += sizeof(struct ibv_flow_spec_action_tag);
1398 case RTE_FLOW_ACTION_TYPE_DROP:
1399 size += sizeof(struct ibv_flow_spec_action_drop);
1401 case RTE_FLOW_ACTION_TYPE_QUEUE:
1403 case RTE_FLOW_ACTION_TYPE_RSS:
1405 case RTE_FLOW_ACTION_TYPE_COUNT:
1406 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1407 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1408 size += sizeof(struct ibv_flow_spec_counter_action);
1419 * Calculate the required bytes that are needed for the item part of the verbs
1423 * Pointer to the list of items.
1426 * The size of the memory needed for all items.
1429 flow_verbs_get_items_size(const struct rte_flow_item items[])
1433 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1434 switch (items->type) {
1435 case RTE_FLOW_ITEM_TYPE_VOID:
1437 case RTE_FLOW_ITEM_TYPE_ETH:
1438 size += sizeof(struct ibv_flow_spec_eth);
1440 case RTE_FLOW_ITEM_TYPE_VLAN:
1441 size += sizeof(struct ibv_flow_spec_eth);
1443 case RTE_FLOW_ITEM_TYPE_IPV4:
1444 size += sizeof(struct ibv_flow_spec_ipv4_ext);
1446 case RTE_FLOW_ITEM_TYPE_IPV6:
1447 size += sizeof(struct ibv_flow_spec_ipv6);
1449 case RTE_FLOW_ITEM_TYPE_UDP:
1450 size += sizeof(struct ibv_flow_spec_tcp_udp);
1452 case RTE_FLOW_ITEM_TYPE_TCP:
1453 size += sizeof(struct ibv_flow_spec_tcp_udp);
1455 case RTE_FLOW_ITEM_TYPE_VXLAN:
1456 size += sizeof(struct ibv_flow_spec_tunnel);
1458 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1459 size += sizeof(struct ibv_flow_spec_tunnel);
1461 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1462 case RTE_FLOW_ITEM_TYPE_GRE:
1463 size += sizeof(struct ibv_flow_spec_gre);
1465 case RTE_FLOW_ITEM_TYPE_MPLS:
1466 size += sizeof(struct ibv_flow_spec_mpls);
1469 case RTE_FLOW_ITEM_TYPE_GRE:
1470 size += sizeof(struct ibv_flow_spec_tunnel);
1481 * Internal preparation function. Allocate mlx5_flow with the required size.
1482 * The required size is calculate based on the actions and items. This function
1483 * also returns the detected actions and items for later use.
1486 * Pointer to Ethernet device.
1488 * Pointer to the flow attributes.
1490 * Pointer to the list of items.
1491 * @param[in] actions
1492 * Pointer to the list of actions.
1494 * Pointer to the error structure.
1497 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1500 static struct mlx5_flow *
1501 flow_verbs_prepare(struct rte_eth_dev *dev,
1502 const struct rte_flow_attr *attr __rte_unused,
1503 const struct rte_flow_item items[],
1504 const struct rte_flow_action actions[],
1505 struct rte_flow_error *error)
1508 struct mlx5_flow *dev_flow;
1509 struct mlx5_flow_handle *dev_handle;
1510 struct mlx5_priv *priv = dev->data->dev_private;
1512 size += flow_verbs_get_actions_size(actions);
1513 size += flow_verbs_get_items_size(items);
1514 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1515 rte_flow_error_set(error, E2BIG,
1516 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1517 "Verbs spec/action size too large");
1520 /* In case of corrupting the memory. */
1521 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1522 rte_flow_error_set(error, ENOSPC,
1523 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1524 "not free temporary device flow");
1527 dev_handle = rte_calloc(__func__, 1, MLX5_FLOW_HANDLE_VERBS_SIZE, 0);
1529 rte_flow_error_set(error, ENOMEM,
1530 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1531 "not enough memory to create flow handle");
1534 /* No multi-thread supporting. */
1535 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1536 dev_flow->handle = dev_handle;
1537 /* Memcpy is used, only size needs to be cleared to 0. */
1538 dev_flow->verbs.size = 0;
1539 dev_flow->verbs.attr.num_of_specs = 0;
1540 dev_flow->ingress = attr->ingress;
1541 /* Need to set transfer attribute: not supported in Verbs mode. */
1546 * Fill the flow with verb spec.
1549 * Pointer to Ethernet device.
1550 * @param[in, out] dev_flow
1551 * Pointer to the mlx5 flow.
1553 * Pointer to the flow attributes.
1555 * Pointer to the list of items.
1556 * @param[in] actions
1557 * Pointer to the list of actions.
1559 * Pointer to the error structure.
1562 * 0 on success, else a negative errno value otherwise and rte_errno is set.
1565 flow_verbs_translate(struct rte_eth_dev *dev,
1566 struct mlx5_flow *dev_flow,
1567 const struct rte_flow_attr *attr,
1568 const struct rte_flow_item items[],
1569 const struct rte_flow_action actions[],
1570 struct rte_flow_error *error)
1572 uint64_t item_flags = 0;
1573 uint64_t action_flags = 0;
1574 uint64_t priority = attr->priority;
1575 uint32_t subpriority = 0;
1576 struct mlx5_priv *priv = dev->data->dev_private;
1578 if (priority == MLX5_FLOW_PRIO_RSVD)
1579 priority = priv->config.flow_prio - 1;
1580 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1583 switch (actions->type) {
1584 case RTE_FLOW_ACTION_TYPE_VOID:
1586 case RTE_FLOW_ACTION_TYPE_FLAG:
1587 flow_verbs_translate_action_flag(dev_flow, actions);
1588 action_flags |= MLX5_FLOW_ACTION_FLAG;
1590 case RTE_FLOW_ACTION_TYPE_MARK:
1591 flow_verbs_translate_action_mark(dev_flow, actions);
1592 action_flags |= MLX5_FLOW_ACTION_MARK;
1594 case RTE_FLOW_ACTION_TYPE_DROP:
1595 flow_verbs_translate_action_drop(dev_flow, actions);
1596 action_flags |= MLX5_FLOW_ACTION_DROP;
1598 case RTE_FLOW_ACTION_TYPE_QUEUE:
1599 flow_verbs_translate_action_queue(dev_flow, actions);
1600 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1602 case RTE_FLOW_ACTION_TYPE_RSS:
1603 flow_verbs_translate_action_rss(dev_flow, actions);
1604 action_flags |= MLX5_FLOW_ACTION_RSS;
1606 case RTE_FLOW_ACTION_TYPE_COUNT:
1607 ret = flow_verbs_translate_action_count(dev_flow,
1612 action_flags |= MLX5_FLOW_ACTION_COUNT;
1615 return rte_flow_error_set(error, ENOTSUP,
1616 RTE_FLOW_ERROR_TYPE_ACTION,
1618 "action not supported");
1621 dev_flow->handle->act_flags = action_flags;
1622 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1623 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1625 switch (items->type) {
1626 case RTE_FLOW_ITEM_TYPE_VOID:
1628 case RTE_FLOW_ITEM_TYPE_ETH:
1629 flow_verbs_translate_item_eth(dev_flow, items,
1631 subpriority = MLX5_PRIORITY_MAP_L2;
1632 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1633 MLX5_FLOW_LAYER_OUTER_L2;
1635 case RTE_FLOW_ITEM_TYPE_VLAN:
1636 flow_verbs_translate_item_vlan(dev_flow, items,
1638 subpriority = MLX5_PRIORITY_MAP_L2;
1639 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1640 MLX5_FLOW_LAYER_INNER_VLAN) :
1641 (MLX5_FLOW_LAYER_OUTER_L2 |
1642 MLX5_FLOW_LAYER_OUTER_VLAN);
1644 case RTE_FLOW_ITEM_TYPE_IPV4:
1645 flow_verbs_translate_item_ipv4(dev_flow, items,
1647 subpriority = MLX5_PRIORITY_MAP_L3;
1648 dev_flow->hash_fields |=
1649 mlx5_flow_hashfields_adjust
1651 MLX5_IPV4_LAYER_TYPES,
1652 MLX5_IPV4_IBV_RX_HASH);
1653 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1654 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1656 case RTE_FLOW_ITEM_TYPE_IPV6:
1657 flow_verbs_translate_item_ipv6(dev_flow, items,
1659 subpriority = MLX5_PRIORITY_MAP_L3;
1660 dev_flow->hash_fields |=
1661 mlx5_flow_hashfields_adjust
1663 MLX5_IPV6_LAYER_TYPES,
1664 MLX5_IPV6_IBV_RX_HASH);
1665 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1666 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1668 case RTE_FLOW_ITEM_TYPE_TCP:
1669 flow_verbs_translate_item_tcp(dev_flow, items,
1671 subpriority = MLX5_PRIORITY_MAP_L4;
1672 dev_flow->hash_fields |=
1673 mlx5_flow_hashfields_adjust
1674 (dev_flow, tunnel, ETH_RSS_TCP,
1675 (IBV_RX_HASH_SRC_PORT_TCP |
1676 IBV_RX_HASH_DST_PORT_TCP));
1677 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1678 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1680 case RTE_FLOW_ITEM_TYPE_UDP:
1681 flow_verbs_translate_item_udp(dev_flow, items,
1683 subpriority = MLX5_PRIORITY_MAP_L4;
1684 dev_flow->hash_fields |=
1685 mlx5_flow_hashfields_adjust
1686 (dev_flow, tunnel, ETH_RSS_UDP,
1687 (IBV_RX_HASH_SRC_PORT_UDP |
1688 IBV_RX_HASH_DST_PORT_UDP));
1689 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1690 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1692 case RTE_FLOW_ITEM_TYPE_VXLAN:
1693 flow_verbs_translate_item_vxlan(dev_flow, items,
1695 subpriority = MLX5_PRIORITY_MAP_L2;
1696 item_flags |= MLX5_FLOW_LAYER_VXLAN;
1698 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1699 flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1701 subpriority = MLX5_PRIORITY_MAP_L2;
1702 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1704 case RTE_FLOW_ITEM_TYPE_GRE:
1705 flow_verbs_translate_item_gre(dev_flow, items,
1707 subpriority = MLX5_PRIORITY_MAP_L2;
1708 item_flags |= MLX5_FLOW_LAYER_GRE;
1710 case RTE_FLOW_ITEM_TYPE_MPLS:
1711 flow_verbs_translate_item_mpls(dev_flow, items,
1713 subpriority = MLX5_PRIORITY_MAP_L2;
1714 item_flags |= MLX5_FLOW_LAYER_MPLS;
1717 return rte_flow_error_set(error, ENOTSUP,
1718 RTE_FLOW_ERROR_TYPE_ITEM,
1720 "item not supported");
1723 dev_flow->handle->layers = item_flags;
1724 /* Other members of attr will be ignored. */
1725 dev_flow->verbs.attr.priority =
1726 mlx5_flow_adjust_priority(dev, priority, subpriority);
1727 dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port;
1732 * Remove the flow from the NIC but keeps it in memory.
1735 * Pointer to the Ethernet device structure.
1736 * @param[in, out] flow
1737 * Pointer to flow structure.
1740 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1742 struct mlx5_flow_handle *handle;
1746 LIST_FOREACH(handle, &flow->dev_handles, next) {
1747 if (handle->ib_flow) {
1748 claim_zero(mlx5_glue->destroy_flow(handle->ib_flow));
1749 handle->ib_flow = NULL;
1752 if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
1753 mlx5_hrxq_drop_release(dev);
1755 mlx5_hrxq_release(dev, handle->hrxq);
1756 handle->hrxq = NULL;
1758 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1759 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1764 * Remove the flow from the NIC and the memory.
1767 * Pointer to the Ethernet device structure.
1768 * @param[in, out] flow
1769 * Pointer to flow structure.
1772 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1774 struct mlx5_flow_handle *handle;
1778 flow_verbs_remove(dev, flow);
1779 while (!LIST_EMPTY(&flow->dev_handles)) {
1780 handle = LIST_FIRST(&flow->dev_handles);
1781 LIST_REMOVE(handle, next);
1784 if (flow->counter) {
1785 flow_verbs_counter_release(dev, flow->counter);
1791 * Apply the flow to the NIC.
1794 * Pointer to the Ethernet device structure.
1795 * @param[in, out] flow
1796 * Pointer to flow structure.
1798 * Pointer to error structure.
1801 * 0 on success, a negative errno value otherwise and rte_errno is set.
1804 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1805 struct rte_flow_error *error)
1807 struct mlx5_priv *priv = dev->data->dev_private;
1808 struct mlx5_flow_handle *handle;
1809 struct mlx5_flow *dev_flow;
1813 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
1814 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1815 handle = dev_flow->handle;
1816 if (handle->act_flags & MLX5_FLOW_ACTION_DROP) {
1817 handle->hrxq = mlx5_hrxq_drop_new(dev);
1818 if (!handle->hrxq) {
1821 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1822 "cannot get drop hash queue");
1826 struct mlx5_hrxq *hrxq;
1828 MLX5_ASSERT(flow->rss.queue);
1829 hrxq = mlx5_hrxq_get(dev, flow->rss.key,
1830 MLX5_RSS_HASH_KEY_LEN,
1831 dev_flow->hash_fields,
1833 flow->rss.queue_num);
1835 hrxq = mlx5_hrxq_new(dev, flow->rss.key,
1836 MLX5_RSS_HASH_KEY_LEN,
1837 dev_flow->hash_fields,
1839 flow->rss.queue_num,
1841 MLX5_FLOW_LAYER_TUNNEL));
1845 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1846 "cannot get hash queue");
1849 handle->hrxq = hrxq;
1851 handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp,
1852 &dev_flow->verbs.attr);
1853 if (!handle->ib_flow) {
1854 rte_flow_error_set(error, errno,
1855 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1857 "hardware refuses to create flow");
1860 if (priv->vmwa_context &&
1861 handle->vf_vlan.tag && !handle->vf_vlan.created) {
1863 * The rule contains the VLAN pattern.
1864 * For VF we are going to create VLAN
1865 * interface to make hypervisor set correct
1866 * e-Switch vport context.
1868 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
1873 err = rte_errno; /* Save rte_errno before cleanup. */
1874 LIST_FOREACH(handle, &flow->dev_handles, next) {
1876 if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
1877 mlx5_hrxq_drop_release(dev);
1879 mlx5_hrxq_release(dev, handle->hrxq);
1880 handle->hrxq = NULL;
1882 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1883 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1885 rte_errno = err; /* Restore rte_errno. */
1892 * @see rte_flow_query()
1896 flow_verbs_query(struct rte_eth_dev *dev,
1897 struct rte_flow *flow,
1898 const struct rte_flow_action *actions,
1900 struct rte_flow_error *error)
1904 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1905 switch (actions->type) {
1906 case RTE_FLOW_ACTION_TYPE_VOID:
1908 case RTE_FLOW_ACTION_TYPE_COUNT:
1909 ret = flow_verbs_counter_query(dev, flow, data, error);
1912 return rte_flow_error_set(error, ENOTSUP,
1913 RTE_FLOW_ERROR_TYPE_ACTION,
1915 "action not supported");
1921 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1922 .validate = flow_verbs_validate,
1923 .prepare = flow_verbs_prepare,
1924 .translate = flow_verbs_translate,
1925 .apply = flow_verbs_apply,
1926 .remove = flow_verbs_remove,
1927 .destroy = flow_verbs_destroy,
1928 .query = flow_verbs_query,