1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
5 #include <netinet/in.h>
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
29 #include <mlx5_glue.h>
32 #include "mlx5_defs.h"
34 #include "mlx5_flow.h"
35 #include "mlx5_rxtx.h"
37 #define VERBS_SPEC_INNER(item_flags) \
38 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
41 * Get Verbs flow counter by index.
44 * Pointer to the Ethernet device structure.
46 * mlx5 flow counter index in the container.
48 * mlx5 flow counter pool in the container,
51 * A pointer to the counter, NULL otherwise.
53 static struct mlx5_flow_counter *
54 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
56 struct mlx5_flow_counter_pool **ppool)
58 struct mlx5_priv *priv = dev->data->dev_private;
59 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
60 struct mlx5_flow_counter_pool *pool;
63 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
67 return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL];
71 * Create Verbs flow counter with Verbs library.
74 * Pointer to the Ethernet device structure.
75 * @param[in, out] counter
76 * mlx5 flow counter object, contains the counter id,
77 * handle of created Verbs flow counter is returned
78 * in cs field (if counters are supported).
81 * 0 On success else a negative errno value is returned
82 * and rte_errno is set.
85 flow_verbs_counter_create(struct rte_eth_dev *dev,
86 struct mlx5_flow_counter *counter)
88 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
89 struct mlx5_priv *priv = dev->data->dev_private;
90 struct ibv_context *ctx = priv->sh->ctx;
91 struct ibv_counter_set_init_attr init = {
92 .counter_set_id = counter->id};
94 counter->cs = mlx5_glue->create_counter_set(ctx, &init);
100 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
101 struct mlx5_priv *priv = dev->data->dev_private;
102 struct ibv_context *ctx = priv->sh->ctx;
103 struct ibv_counters_init_attr init = {0};
104 struct ibv_counter_attach_attr attach;
107 memset(&attach, 0, sizeof(attach));
108 counter->cs = mlx5_glue->create_counters(ctx, &init);
113 attach.counter_desc = IBV_COUNTER_PACKETS;
115 ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
117 attach.counter_desc = IBV_COUNTER_BYTES;
119 ret = mlx5_glue->attach_counters
120 (counter->cs, &attach, NULL);
123 claim_zero(mlx5_glue->destroy_counters(counter->cs));
138 * Get a flow counter.
141 * Pointer to the Ethernet device structure.
143 * Indicate if this counter is shared with other flows.
145 * Counter identifier.
148 * Index to the counter, 0 otherwise and rte_errno is set.
151 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
153 struct mlx5_priv *priv = dev->data->dev_private;
154 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0);
155 struct mlx5_flow_counter_pool *pool = NULL;
156 struct mlx5_flow_counter *cnt = NULL;
157 uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
163 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
164 pool = cont->pools[pool_idx];
165 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
166 cnt = &pool->counters_raw[i];
167 if (cnt->shared && cnt->id == id) {
169 return MLX5_MAKE_CNT_IDX(pool_idx, i);
174 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
175 pool = cont->pools[pool_idx];
178 cnt = TAILQ_FIRST(&pool->counters);
183 struct mlx5_flow_counter_pool **pools;
186 if (n_valid == cont->n) {
187 /* Resize the container pool array. */
188 size = sizeof(struct mlx5_flow_counter_pool *) *
189 (n_valid + MLX5_CNT_CONTAINER_RESIZE);
190 pools = rte_zmalloc(__func__, size, 0);
194 memcpy(pools, cont->pools,
195 sizeof(struct mlx5_flow_counter_pool *) *
197 rte_free(cont->pools);
200 cont->n += MLX5_CNT_CONTAINER_RESIZE;
202 /* Allocate memory for new pool*/
203 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
204 pool = rte_calloc(__func__, 1, size, 0);
207 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
208 cnt = &pool->counters_raw[i];
209 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
211 cnt = &pool->counters_raw[0];
212 cont->pools[n_valid] = pool;
214 rte_atomic16_add(&cont->n_valid, 1);
215 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
218 cnt->shared = shared;
222 /* Create counter with Verbs. */
223 ret = flow_verbs_counter_create(dev, cnt);
225 TAILQ_REMOVE(&pool->counters, cnt, next);
226 return MLX5_MAKE_CNT_IDX(pool_idx, (cnt - pool->counters_raw));
228 /* Some error occurred in Verbs library. */
234 * Release a flow counter.
237 * Pointer to the Ethernet device structure.
239 * Index to the counter handler.
242 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
244 struct mlx5_flow_counter_pool *pool;
245 struct mlx5_flow_counter *cnt;
247 cnt = flow_verbs_counter_get_by_idx(dev, counter,
249 if (--cnt->ref_cnt == 0) {
250 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
251 claim_zero(mlx5_glue->destroy_counter_set(cnt->cs));
253 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
254 claim_zero(mlx5_glue->destroy_counters(cnt->cs));
257 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
262 * Query a flow counter via Verbs library call.
264 * @see rte_flow_query()
268 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
269 struct rte_flow *flow, void *data,
270 struct rte_flow_error *error)
272 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
273 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
275 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
276 (dev, flow->counter, NULL);
277 struct rte_flow_query_count *qc = data;
278 uint64_t counters[2] = {0, 0};
279 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
280 struct ibv_query_counter_set_attr query_cs_attr = {
282 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
284 struct ibv_counter_set_data query_out = {
286 .outlen = 2 * sizeof(uint64_t),
288 int err = mlx5_glue->query_counter_set(&query_cs_attr,
290 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
291 int err = mlx5_glue->query_counters
294 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
297 return rte_flow_error_set
299 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
301 "cannot read counter");
304 qc->hits = counters[0] - cnt->hits;
305 qc->bytes = counters[1] - cnt->bytes;
307 cnt->hits = counters[0];
308 cnt->bytes = counters[1];
312 return rte_flow_error_set(error, EINVAL,
313 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
315 "flow does not have counter");
319 return rte_flow_error_set(error, ENOTSUP,
320 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
322 "counters are not available");
327 * Add a verbs item specification into @p verbs.
330 * Pointer to verbs structure.
332 * Create specification.
334 * Size in bytes of the specification to copy.
337 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
338 void *src, unsigned int size)
344 MLX5_ASSERT(verbs->specs);
345 dst = (void *)(verbs->specs + verbs->size);
346 memcpy(dst, src, size);
347 ++verbs->attr.num_of_specs;
352 * Convert the @p item into a Verbs specification. This function assumes that
353 * the input is valid and that there is space to insert the requested item
356 * @param[in, out] dev_flow
357 * Pointer to dev_flow structure.
359 * Item specification.
360 * @param[in] item_flags
364 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
365 const struct rte_flow_item *item,
368 const struct rte_flow_item_eth *spec = item->spec;
369 const struct rte_flow_item_eth *mask = item->mask;
370 const unsigned int size = sizeof(struct ibv_flow_spec_eth);
371 struct ibv_flow_spec_eth eth = {
372 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
377 mask = &rte_flow_item_eth_mask;
381 memcpy(ð.val.dst_mac, spec->dst.addr_bytes,
383 memcpy(ð.val.src_mac, spec->src.addr_bytes,
385 eth.val.ether_type = spec->type;
386 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes,
388 memcpy(ð.mask.src_mac, mask->src.addr_bytes,
390 eth.mask.ether_type = mask->type;
391 /* Remove unwanted bits from values. */
392 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
393 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
394 eth.val.src_mac[i] &= eth.mask.src_mac[i];
396 eth.val.ether_type &= eth.mask.ether_type;
398 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
402 * Update the VLAN tag in the Verbs Ethernet specification.
403 * This function assumes that the input is valid and there is space to add
404 * the requested item.
406 * @param[in, out] attr
407 * Pointer to Verbs attributes structure.
409 * Verbs structure containing the VLAN information to copy.
412 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
413 struct ibv_flow_spec_eth *eth)
416 const enum ibv_flow_spec_type search = eth->type;
417 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
418 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
420 for (i = 0; i != attr->num_of_specs; ++i) {
421 if (hdr->type == search) {
422 struct ibv_flow_spec_eth *e =
423 (struct ibv_flow_spec_eth *)hdr;
425 e->val.vlan_tag = eth->val.vlan_tag;
426 e->mask.vlan_tag = eth->mask.vlan_tag;
427 e->val.ether_type = eth->val.ether_type;
428 e->mask.ether_type = eth->mask.ether_type;
431 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
436 * Convert the @p item into a Verbs specification. This function assumes that
437 * the input is valid and that there is space to insert the requested item
440 * @param[in, out] dev_flow
441 * Pointer to dev_flow structure.
443 * Item specification.
444 * @param[in] item_flags
448 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
449 const struct rte_flow_item *item,
452 const struct rte_flow_item_vlan *spec = item->spec;
453 const struct rte_flow_item_vlan *mask = item->mask;
454 unsigned int size = sizeof(struct ibv_flow_spec_eth);
455 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
456 struct ibv_flow_spec_eth eth = {
457 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
460 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
461 MLX5_FLOW_LAYER_OUTER_L2;
464 mask = &rte_flow_item_vlan_mask;
466 eth.val.vlan_tag = spec->tci;
467 eth.mask.vlan_tag = mask->tci;
468 eth.val.vlan_tag &= eth.mask.vlan_tag;
469 eth.val.ether_type = spec->inner_type;
470 eth.mask.ether_type = mask->inner_type;
471 eth.val.ether_type &= eth.mask.ether_type;
473 if (!(item_flags & l2m))
474 flow_verbs_spec_add(&dev_flow->verbs, ð, size);
476 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð);
478 dev_flow->handle->vf_vlan.tag =
479 rte_be_to_cpu_16(spec->tci) & 0x0fff;
483 * Convert the @p item into a Verbs specification. This function assumes that
484 * the input is valid and that there is space to insert the requested item
487 * @param[in, out] dev_flow
488 * Pointer to dev_flow structure.
490 * Item specification.
491 * @param[in] item_flags
495 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
496 const struct rte_flow_item *item,
499 const struct rte_flow_item_ipv4 *spec = item->spec;
500 const struct rte_flow_item_ipv4 *mask = item->mask;
501 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
502 struct ibv_flow_spec_ipv4_ext ipv4 = {
503 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
508 mask = &rte_flow_item_ipv4_mask;
510 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
511 .src_ip = spec->hdr.src_addr,
512 .dst_ip = spec->hdr.dst_addr,
513 .proto = spec->hdr.next_proto_id,
514 .tos = spec->hdr.type_of_service,
516 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
517 .src_ip = mask->hdr.src_addr,
518 .dst_ip = mask->hdr.dst_addr,
519 .proto = mask->hdr.next_proto_id,
520 .tos = mask->hdr.type_of_service,
522 /* Remove unwanted bits from values. */
523 ipv4.val.src_ip &= ipv4.mask.src_ip;
524 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
525 ipv4.val.proto &= ipv4.mask.proto;
526 ipv4.val.tos &= ipv4.mask.tos;
528 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
532 * Convert the @p item into a Verbs specification. This function assumes that
533 * the input is valid and that there is space to insert the requested item
536 * @param[in, out] dev_flow
537 * Pointer to dev_flow structure.
539 * Item specification.
540 * @param[in] item_flags
544 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
545 const struct rte_flow_item *item,
548 const struct rte_flow_item_ipv6 *spec = item->spec;
549 const struct rte_flow_item_ipv6 *mask = item->mask;
550 unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
551 struct ibv_flow_spec_ipv6 ipv6 = {
552 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
557 mask = &rte_flow_item_ipv6_mask;
560 uint32_t vtc_flow_val;
561 uint32_t vtc_flow_mask;
563 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
564 RTE_DIM(ipv6.val.src_ip));
565 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
566 RTE_DIM(ipv6.val.dst_ip));
567 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
568 RTE_DIM(ipv6.mask.src_ip));
569 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
570 RTE_DIM(ipv6.mask.dst_ip));
571 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
572 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
573 ipv6.val.flow_label =
574 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
575 RTE_IPV6_HDR_FL_SHIFT);
576 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
577 RTE_IPV6_HDR_TC_SHIFT;
578 ipv6.val.next_hdr = spec->hdr.proto;
579 ipv6.mask.flow_label =
580 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
581 RTE_IPV6_HDR_FL_SHIFT);
582 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
583 RTE_IPV6_HDR_TC_SHIFT;
584 ipv6.mask.next_hdr = mask->hdr.proto;
585 /* Remove unwanted bits from values. */
586 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
587 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
588 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
590 ipv6.val.flow_label &= ipv6.mask.flow_label;
591 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
592 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
594 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
598 * Convert the @p item into a Verbs specification. This function assumes that
599 * the input is valid and that there is space to insert the requested item
602 * @param[in, out] dev_flow
603 * Pointer to dev_flow structure.
605 * Item specification.
606 * @param[in] item_flags
610 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
611 const struct rte_flow_item *item,
612 uint64_t item_flags __rte_unused)
614 const struct rte_flow_item_tcp *spec = item->spec;
615 const struct rte_flow_item_tcp *mask = item->mask;
616 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
617 struct ibv_flow_spec_tcp_udp tcp = {
618 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
623 mask = &rte_flow_item_tcp_mask;
625 tcp.val.dst_port = spec->hdr.dst_port;
626 tcp.val.src_port = spec->hdr.src_port;
627 tcp.mask.dst_port = mask->hdr.dst_port;
628 tcp.mask.src_port = mask->hdr.src_port;
629 /* Remove unwanted bits from values. */
630 tcp.val.src_port &= tcp.mask.src_port;
631 tcp.val.dst_port &= tcp.mask.dst_port;
633 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
637 * Convert the @p item into a Verbs specification. This function assumes that
638 * the input is valid and that there is space to insert the requested item
641 * @param[in, out] dev_flow
642 * Pointer to dev_flow structure.
644 * Item specification.
645 * @param[in] item_flags
649 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
650 const struct rte_flow_item *item,
651 uint64_t item_flags __rte_unused)
653 const struct rte_flow_item_udp *spec = item->spec;
654 const struct rte_flow_item_udp *mask = item->mask;
655 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
656 struct ibv_flow_spec_tcp_udp udp = {
657 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
662 mask = &rte_flow_item_udp_mask;
664 udp.val.dst_port = spec->hdr.dst_port;
665 udp.val.src_port = spec->hdr.src_port;
666 udp.mask.dst_port = mask->hdr.dst_port;
667 udp.mask.src_port = mask->hdr.src_port;
668 /* Remove unwanted bits from values. */
669 udp.val.src_port &= udp.mask.src_port;
670 udp.val.dst_port &= udp.mask.dst_port;
672 flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
676 * Convert the @p item into a Verbs specification. This function assumes that
677 * the input is valid and that there is space to insert the requested item
680 * @param[in, out] dev_flow
681 * Pointer to dev_flow structure.
683 * Item specification.
684 * @param[in] item_flags
688 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
689 const struct rte_flow_item *item,
690 uint64_t item_flags __rte_unused)
692 const struct rte_flow_item_vxlan *spec = item->spec;
693 const struct rte_flow_item_vxlan *mask = item->mask;
694 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
695 struct ibv_flow_spec_tunnel vxlan = {
696 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
702 } id = { .vlan_id = 0, };
705 mask = &rte_flow_item_vxlan_mask;
707 memcpy(&id.vni[1], spec->vni, 3);
708 vxlan.val.tunnel_id = id.vlan_id;
709 memcpy(&id.vni[1], mask->vni, 3);
710 vxlan.mask.tunnel_id = id.vlan_id;
711 /* Remove unwanted bits from values. */
712 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
714 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
718 * Convert the @p item into a Verbs specification. This function assumes that
719 * the input is valid and that there is space to insert the requested item
722 * @param[in, out] dev_flow
723 * Pointer to dev_flow structure.
725 * Item specification.
726 * @param[in] item_flags
730 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
731 const struct rte_flow_item *item,
732 uint64_t item_flags __rte_unused)
734 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
735 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
736 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
737 struct ibv_flow_spec_tunnel vxlan_gpe = {
738 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
744 } id = { .vlan_id = 0, };
747 mask = &rte_flow_item_vxlan_gpe_mask;
749 memcpy(&id.vni[1], spec->vni, 3);
750 vxlan_gpe.val.tunnel_id = id.vlan_id;
751 memcpy(&id.vni[1], mask->vni, 3);
752 vxlan_gpe.mask.tunnel_id = id.vlan_id;
753 /* Remove unwanted bits from values. */
754 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
756 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
760 * Update the protocol in Verbs IPv4/IPv6 spec.
762 * @param[in, out] attr
763 * Pointer to Verbs attributes structure.
765 * Specification type to search in order to update the IP protocol.
766 * @param[in] protocol
767 * Protocol value to set if none is present in the specification.
770 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
771 enum ibv_flow_spec_type search,
775 struct ibv_spec_header *hdr = (struct ibv_spec_header *)
776 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
780 for (i = 0; i != attr->num_of_specs; ++i) {
781 if (hdr->type == search) {
783 struct ibv_flow_spec_ipv4_ext *ipv4;
784 struct ibv_flow_spec_ipv6 *ipv6;
788 case IBV_FLOW_SPEC_IPV4_EXT:
789 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
790 if (!ip.ipv4->val.proto) {
791 ip.ipv4->val.proto = protocol;
792 ip.ipv4->mask.proto = 0xff;
795 case IBV_FLOW_SPEC_IPV6:
796 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
797 if (!ip.ipv6->val.next_hdr) {
798 ip.ipv6->val.next_hdr = protocol;
799 ip.ipv6->mask.next_hdr = 0xff;
807 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
812 * Convert the @p item into a Verbs specification. This function assumes that
813 * the input is valid and that there is space to insert the requested item
816 * @param[in, out] dev_flow
817 * Pointer to dev_flow structure.
819 * Item specification.
820 * @param[in] item_flags
824 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
825 const struct rte_flow_item *item __rte_unused,
828 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
829 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
830 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
831 struct ibv_flow_spec_tunnel tunnel = {
832 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
836 const struct rte_flow_item_gre *spec = item->spec;
837 const struct rte_flow_item_gre *mask = item->mask;
838 unsigned int size = sizeof(struct ibv_flow_spec_gre);
839 struct ibv_flow_spec_gre tunnel = {
840 .type = IBV_FLOW_SPEC_GRE,
845 mask = &rte_flow_item_gre_mask;
847 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
848 tunnel.val.protocol = spec->protocol;
849 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
850 tunnel.mask.protocol = mask->protocol;
851 /* Remove unwanted bits from values. */
852 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
853 tunnel.val.protocol &= tunnel.mask.protocol;
854 tunnel.val.key &= tunnel.mask.key;
857 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
858 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
859 IBV_FLOW_SPEC_IPV4_EXT,
862 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
865 flow_verbs_spec_add(verbs, &tunnel, size);
869 * Convert the @p action into a Verbs specification. This function assumes that
870 * the input is valid and that there is space to insert the requested action
871 * into the flow. This function also return the action that was added.
873 * @param[in, out] dev_flow
874 * Pointer to dev_flow structure.
876 * Item specification.
877 * @param[in] item_flags
881 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
882 const struct rte_flow_item *item __rte_unused,
883 uint64_t item_flags __rte_unused)
885 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
886 const struct rte_flow_item_mpls *spec = item->spec;
887 const struct rte_flow_item_mpls *mask = item->mask;
888 unsigned int size = sizeof(struct ibv_flow_spec_mpls);
889 struct ibv_flow_spec_mpls mpls = {
890 .type = IBV_FLOW_SPEC_MPLS,
895 mask = &rte_flow_item_mpls_mask;
897 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
898 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
899 /* Remove unwanted bits from values. */
900 mpls.val.label &= mpls.mask.label;
902 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
907 * Convert the @p action into a Verbs specification. This function assumes that
908 * the input is valid and that there is space to insert the requested action
911 * @param[in] dev_flow
912 * Pointer to mlx5_flow.
914 * Action configuration.
917 flow_verbs_translate_action_drop
918 (struct mlx5_flow *dev_flow,
919 const struct rte_flow_action *action __rte_unused)
921 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
922 struct ibv_flow_spec_action_drop drop = {
923 .type = IBV_FLOW_SPEC_ACTION_DROP,
927 flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
931 * Convert the @p action into a Verbs specification. This function assumes that
932 * the input is valid and that there is space to insert the requested action
935 * @param[in] dev_flow
936 * Pointer to mlx5_flow.
938 * Action configuration.
941 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
942 const struct rte_flow_action *action)
944 const struct rte_flow_action_queue *queue = action->conf;
945 struct rte_flow *flow = dev_flow->flow;
948 (*flow->rss.queue)[0] = queue->index;
949 flow->rss.queue_num = 1;
953 * Convert the @p action into a Verbs specification. This function assumes that
954 * the input is valid and that there is space to insert the requested action
958 * Action configuration.
959 * @param[in, out] action_flags
960 * Pointer to the detected actions.
961 * @param[in] dev_flow
962 * Pointer to mlx5_flow.
965 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
966 const struct rte_flow_action *action)
968 const struct rte_flow_action_rss *rss = action->conf;
969 const uint8_t *rss_key;
970 struct rte_flow *flow = dev_flow->flow;
973 memcpy((*flow->rss.queue), rss->queue,
974 rss->queue_num * sizeof(uint16_t));
975 flow->rss.queue_num = rss->queue_num;
976 /* NULL RSS key indicates default RSS key. */
977 rss_key = !rss->key ? rss_hash_default_key : rss->key;
978 memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
980 * rss->level and rss.types should be set in advance when expanding
986 * Convert the @p action into a Verbs specification. This function assumes that
987 * the input is valid and that there is space to insert the requested action
990 * @param[in] dev_flow
991 * Pointer to mlx5_flow.
993 * Action configuration.
996 flow_verbs_translate_action_flag
997 (struct mlx5_flow *dev_flow,
998 const struct rte_flow_action *action __rte_unused)
1000 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1001 struct ibv_flow_spec_action_tag tag = {
1002 .type = IBV_FLOW_SPEC_ACTION_TAG,
1004 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1007 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1011 * Convert the @p action into a Verbs specification. This function assumes that
1012 * the input is valid and that there is space to insert the requested action
1015 * @param[in] dev_flow
1016 * Pointer to mlx5_flow.
1018 * Action configuration.
1021 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1022 const struct rte_flow_action *action)
1024 const struct rte_flow_action_mark *mark = action->conf;
1025 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1026 struct ibv_flow_spec_action_tag tag = {
1027 .type = IBV_FLOW_SPEC_ACTION_TAG,
1029 .tag_id = mlx5_flow_mark_set(mark->id),
1032 flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1036 * Convert the @p action into a Verbs specification. This function assumes that
1037 * the input is valid and that there is space to insert the requested action
1041 * Pointer to the Ethernet device structure.
1043 * Action configuration.
1044 * @param[in] dev_flow
1045 * Pointer to mlx5_flow.
1047 * Pointer to error structure.
1050 * 0 On success else a negative errno value is returned and rte_errno is set.
1053 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1054 const struct rte_flow_action *action,
1055 struct rte_eth_dev *dev,
1056 struct rte_flow_error *error)
1058 const struct rte_flow_action_count *count = action->conf;
1059 struct rte_flow *flow = dev_flow->flow;
1060 struct mlx5_flow_counter *cnt = NULL;
1061 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1062 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1063 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1064 struct ibv_flow_spec_counter_action counter = {
1065 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1070 if (!flow->counter) {
1071 flow->counter = flow_verbs_counter_new(dev, count->shared,
1074 return rte_flow_error_set(error, rte_errno,
1075 RTE_FLOW_ERROR_TYPE_ACTION,
1077 "cannot get counter"
1080 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, NULL);
1081 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1082 counter.counter_set_handle = cnt->cs->handle;
1083 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1084 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1085 counter.counters = cnt->cs;
1086 flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1092 * Internal validation function. For validating both actions and items.
1095 * Pointer to the Ethernet device structure.
1097 * Pointer to the flow attributes.
1099 * Pointer to the list of items.
1100 * @param[in] actions
1101 * Pointer to the list of actions.
1102 * @param[in] external
1103 * This flow rule is created by request external to PMD.
1105 * Pointer to the error structure.
1108 * 0 on success, a negative errno value otherwise and rte_errno is set.
1111 flow_verbs_validate(struct rte_eth_dev *dev,
1112 const struct rte_flow_attr *attr,
1113 const struct rte_flow_item items[],
1114 const struct rte_flow_action actions[],
1115 bool external __rte_unused,
1116 struct rte_flow_error *error)
1119 uint64_t action_flags = 0;
1120 uint64_t item_flags = 0;
1121 uint64_t last_item = 0;
1122 uint8_t next_protocol = 0xff;
1123 uint16_t ether_type = 0;
1127 ret = mlx5_flow_validate_attributes(dev, attr, error);
1130 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1131 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1134 switch (items->type) {
1135 case RTE_FLOW_ITEM_TYPE_VOID:
1137 case RTE_FLOW_ITEM_TYPE_ETH:
1138 ret = mlx5_flow_validate_item_eth(items, item_flags,
1142 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1143 MLX5_FLOW_LAYER_OUTER_L2;
1144 if (items->mask != NULL && items->spec != NULL) {
1146 ((const struct rte_flow_item_eth *)
1149 ((const struct rte_flow_item_eth *)
1151 ether_type = rte_be_to_cpu_16(ether_type);
1156 case RTE_FLOW_ITEM_TYPE_VLAN:
1157 ret = mlx5_flow_validate_item_vlan(items, item_flags,
1161 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1162 MLX5_FLOW_LAYER_INNER_VLAN) :
1163 (MLX5_FLOW_LAYER_OUTER_L2 |
1164 MLX5_FLOW_LAYER_OUTER_VLAN);
1165 if (items->mask != NULL && items->spec != NULL) {
1167 ((const struct rte_flow_item_vlan *)
1168 items->spec)->inner_type;
1170 ((const struct rte_flow_item_vlan *)
1171 items->mask)->inner_type;
1172 ether_type = rte_be_to_cpu_16(ether_type);
1177 case RTE_FLOW_ITEM_TYPE_IPV4:
1178 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1184 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1185 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1186 if (items->mask != NULL &&
1187 ((const struct rte_flow_item_ipv4 *)
1188 items->mask)->hdr.next_proto_id) {
1190 ((const struct rte_flow_item_ipv4 *)
1191 (items->spec))->hdr.next_proto_id;
1193 ((const struct rte_flow_item_ipv4 *)
1194 (items->mask))->hdr.next_proto_id;
1196 /* Reset for inner layer. */
1197 next_protocol = 0xff;
1200 case RTE_FLOW_ITEM_TYPE_IPV6:
1201 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1207 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1208 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1209 if (items->mask != NULL &&
1210 ((const struct rte_flow_item_ipv6 *)
1211 items->mask)->hdr.proto) {
1213 ((const struct rte_flow_item_ipv6 *)
1214 items->spec)->hdr.proto;
1216 ((const struct rte_flow_item_ipv6 *)
1217 items->mask)->hdr.proto;
1219 /* Reset for inner layer. */
1220 next_protocol = 0xff;
1223 case RTE_FLOW_ITEM_TYPE_UDP:
1224 ret = mlx5_flow_validate_item_udp(items, item_flags,
1229 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1230 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1232 case RTE_FLOW_ITEM_TYPE_TCP:
1233 ret = mlx5_flow_validate_item_tcp
1236 &rte_flow_item_tcp_mask,
1240 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1241 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1243 case RTE_FLOW_ITEM_TYPE_VXLAN:
1244 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1248 last_item = MLX5_FLOW_LAYER_VXLAN;
1250 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1251 ret = mlx5_flow_validate_item_vxlan_gpe(items,
1256 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1258 case RTE_FLOW_ITEM_TYPE_GRE:
1259 ret = mlx5_flow_validate_item_gre(items, item_flags,
1260 next_protocol, error);
1263 last_item = MLX5_FLOW_LAYER_GRE;
1265 case RTE_FLOW_ITEM_TYPE_MPLS:
1266 ret = mlx5_flow_validate_item_mpls(dev, items,
1271 last_item = MLX5_FLOW_LAYER_MPLS;
1274 return rte_flow_error_set(error, ENOTSUP,
1275 RTE_FLOW_ERROR_TYPE_ITEM,
1276 NULL, "item not supported");
1278 item_flags |= last_item;
1280 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1281 switch (actions->type) {
1282 case RTE_FLOW_ACTION_TYPE_VOID:
1284 case RTE_FLOW_ACTION_TYPE_FLAG:
1285 ret = mlx5_flow_validate_action_flag(action_flags,
1290 action_flags |= MLX5_FLOW_ACTION_FLAG;
1292 case RTE_FLOW_ACTION_TYPE_MARK:
1293 ret = mlx5_flow_validate_action_mark(actions,
1299 action_flags |= MLX5_FLOW_ACTION_MARK;
1301 case RTE_FLOW_ACTION_TYPE_DROP:
1302 ret = mlx5_flow_validate_action_drop(action_flags,
1307 action_flags |= MLX5_FLOW_ACTION_DROP;
1309 case RTE_FLOW_ACTION_TYPE_QUEUE:
1310 ret = mlx5_flow_validate_action_queue(actions,
1316 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1318 case RTE_FLOW_ACTION_TYPE_RSS:
1319 ret = mlx5_flow_validate_action_rss(actions,
1325 action_flags |= MLX5_FLOW_ACTION_RSS;
1327 case RTE_FLOW_ACTION_TYPE_COUNT:
1328 ret = mlx5_flow_validate_action_count(dev, attr, error);
1331 action_flags |= MLX5_FLOW_ACTION_COUNT;
1334 return rte_flow_error_set(error, ENOTSUP,
1335 RTE_FLOW_ERROR_TYPE_ACTION,
1337 "action not supported");
1341 * Validate the drop action mutual exclusion with other actions.
1342 * Drop action is mutually-exclusive with any other action, except for
1345 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1346 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1347 return rte_flow_error_set(error, EINVAL,
1348 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1349 "Drop action is mutually-exclusive "
1350 "with any other action, except for "
1352 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1353 return rte_flow_error_set(error, EINVAL,
1354 RTE_FLOW_ERROR_TYPE_ACTION, actions,
1355 "no fate action is found");
1360 * Calculate the required bytes that are needed for the action part of the verbs
1363 * @param[in] actions
1364 * Pointer to the list of actions.
1367 * The size of the memory needed for all actions.
1370 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1374 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1375 switch (actions->type) {
1376 case RTE_FLOW_ACTION_TYPE_VOID:
1378 case RTE_FLOW_ACTION_TYPE_FLAG:
1379 size += sizeof(struct ibv_flow_spec_action_tag);
1381 case RTE_FLOW_ACTION_TYPE_MARK:
1382 size += sizeof(struct ibv_flow_spec_action_tag);
1384 case RTE_FLOW_ACTION_TYPE_DROP:
1385 size += sizeof(struct ibv_flow_spec_action_drop);
1387 case RTE_FLOW_ACTION_TYPE_QUEUE:
1389 case RTE_FLOW_ACTION_TYPE_RSS:
1391 case RTE_FLOW_ACTION_TYPE_COUNT:
1392 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1393 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1394 size += sizeof(struct ibv_flow_spec_counter_action);
1405 * Calculate the required bytes that are needed for the item part of the verbs
1409 * Pointer to the list of items.
1412 * The size of the memory needed for all items.
1415 flow_verbs_get_items_size(const struct rte_flow_item items[])
1419 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1420 switch (items->type) {
1421 case RTE_FLOW_ITEM_TYPE_VOID:
1423 case RTE_FLOW_ITEM_TYPE_ETH:
1424 size += sizeof(struct ibv_flow_spec_eth);
1426 case RTE_FLOW_ITEM_TYPE_VLAN:
1427 size += sizeof(struct ibv_flow_spec_eth);
1429 case RTE_FLOW_ITEM_TYPE_IPV4:
1430 size += sizeof(struct ibv_flow_spec_ipv4_ext);
1432 case RTE_FLOW_ITEM_TYPE_IPV6:
1433 size += sizeof(struct ibv_flow_spec_ipv6);
1435 case RTE_FLOW_ITEM_TYPE_UDP:
1436 size += sizeof(struct ibv_flow_spec_tcp_udp);
1438 case RTE_FLOW_ITEM_TYPE_TCP:
1439 size += sizeof(struct ibv_flow_spec_tcp_udp);
1441 case RTE_FLOW_ITEM_TYPE_VXLAN:
1442 size += sizeof(struct ibv_flow_spec_tunnel);
1444 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1445 size += sizeof(struct ibv_flow_spec_tunnel);
1447 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1448 case RTE_FLOW_ITEM_TYPE_GRE:
1449 size += sizeof(struct ibv_flow_spec_gre);
1451 case RTE_FLOW_ITEM_TYPE_MPLS:
1452 size += sizeof(struct ibv_flow_spec_mpls);
1455 case RTE_FLOW_ITEM_TYPE_GRE:
1456 size += sizeof(struct ibv_flow_spec_tunnel);
1467 * Internal preparation function. Allocate mlx5_flow with the required size.
1468 * The required size is calculate based on the actions and items. This function
1469 * also returns the detected actions and items for later use.
1472 * Pointer to Ethernet device.
1474 * Pointer to the flow attributes.
1476 * Pointer to the list of items.
1477 * @param[in] actions
1478 * Pointer to the list of actions.
1480 * Pointer to the error structure.
1483 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1486 static struct mlx5_flow *
1487 flow_verbs_prepare(struct rte_eth_dev *dev,
1488 const struct rte_flow_attr *attr __rte_unused,
1489 const struct rte_flow_item items[],
1490 const struct rte_flow_action actions[],
1491 struct rte_flow_error *error)
1494 struct mlx5_flow *dev_flow;
1495 struct mlx5_flow_handle *dev_handle;
1496 struct mlx5_priv *priv = dev->data->dev_private;
1498 size += flow_verbs_get_actions_size(actions);
1499 size += flow_verbs_get_items_size(items);
1500 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1501 rte_flow_error_set(error, E2BIG,
1502 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1503 "Verbs spec/action size too large");
1506 /* In case of corrupting the memory. */
1507 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1508 rte_flow_error_set(error, ENOSPC,
1509 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1510 "not free temporary device flow");
1513 dev_handle = rte_calloc(__func__, 1, MLX5_FLOW_HANDLE_VERBS_SIZE, 0);
1515 rte_flow_error_set(error, ENOMEM,
1516 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1517 "not enough memory to create flow handle");
1520 /* No multi-thread supporting. */
1521 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1522 dev_flow->handle = dev_handle;
1523 /* Memcpy is used, only size needs to be cleared to 0. */
1524 dev_flow->verbs.size = 0;
1525 dev_flow->verbs.attr.num_of_specs = 0;
1526 dev_flow->ingress = attr->ingress;
1527 /* Need to set transfer attribute: not supported in Verbs mode. */
1532 * Fill the flow with verb spec.
1535 * Pointer to Ethernet device.
1536 * @param[in, out] dev_flow
1537 * Pointer to the mlx5 flow.
1539 * Pointer to the flow attributes.
1541 * Pointer to the list of items.
1542 * @param[in] actions
1543 * Pointer to the list of actions.
1545 * Pointer to the error structure.
1548 * 0 on success, else a negative errno value otherwise and rte_errno is set.
1551 flow_verbs_translate(struct rte_eth_dev *dev,
1552 struct mlx5_flow *dev_flow,
1553 const struct rte_flow_attr *attr,
1554 const struct rte_flow_item items[],
1555 const struct rte_flow_action actions[],
1556 struct rte_flow_error *error)
1558 uint64_t item_flags = 0;
1559 uint64_t action_flags = 0;
1560 uint64_t priority = attr->priority;
1561 uint32_t subpriority = 0;
1562 struct mlx5_priv *priv = dev->data->dev_private;
1564 if (priority == MLX5_FLOW_PRIO_RSVD)
1565 priority = priv->config.flow_prio - 1;
1566 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1569 switch (actions->type) {
1570 case RTE_FLOW_ACTION_TYPE_VOID:
1572 case RTE_FLOW_ACTION_TYPE_FLAG:
1573 flow_verbs_translate_action_flag(dev_flow, actions);
1574 action_flags |= MLX5_FLOW_ACTION_FLAG;
1576 case RTE_FLOW_ACTION_TYPE_MARK:
1577 flow_verbs_translate_action_mark(dev_flow, actions);
1578 action_flags |= MLX5_FLOW_ACTION_MARK;
1580 case RTE_FLOW_ACTION_TYPE_DROP:
1581 flow_verbs_translate_action_drop(dev_flow, actions);
1582 action_flags |= MLX5_FLOW_ACTION_DROP;
1584 case RTE_FLOW_ACTION_TYPE_QUEUE:
1585 flow_verbs_translate_action_queue(dev_flow, actions);
1586 action_flags |= MLX5_FLOW_ACTION_QUEUE;
1588 case RTE_FLOW_ACTION_TYPE_RSS:
1589 flow_verbs_translate_action_rss(dev_flow, actions);
1590 action_flags |= MLX5_FLOW_ACTION_RSS;
1592 case RTE_FLOW_ACTION_TYPE_COUNT:
1593 ret = flow_verbs_translate_action_count(dev_flow,
1598 action_flags |= MLX5_FLOW_ACTION_COUNT;
1601 return rte_flow_error_set(error, ENOTSUP,
1602 RTE_FLOW_ERROR_TYPE_ACTION,
1604 "action not supported");
1607 dev_flow->handle->act_flags = action_flags;
1608 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1609 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1611 switch (items->type) {
1612 case RTE_FLOW_ITEM_TYPE_VOID:
1614 case RTE_FLOW_ITEM_TYPE_ETH:
1615 flow_verbs_translate_item_eth(dev_flow, items,
1617 subpriority = MLX5_PRIORITY_MAP_L2;
1618 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1619 MLX5_FLOW_LAYER_OUTER_L2;
1621 case RTE_FLOW_ITEM_TYPE_VLAN:
1622 flow_verbs_translate_item_vlan(dev_flow, items,
1624 subpriority = MLX5_PRIORITY_MAP_L2;
1625 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1626 MLX5_FLOW_LAYER_INNER_VLAN) :
1627 (MLX5_FLOW_LAYER_OUTER_L2 |
1628 MLX5_FLOW_LAYER_OUTER_VLAN);
1630 case RTE_FLOW_ITEM_TYPE_IPV4:
1631 flow_verbs_translate_item_ipv4(dev_flow, items,
1633 subpriority = MLX5_PRIORITY_MAP_L3;
1634 dev_flow->hash_fields |=
1635 mlx5_flow_hashfields_adjust
1637 MLX5_IPV4_LAYER_TYPES,
1638 MLX5_IPV4_IBV_RX_HASH);
1639 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1640 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1642 case RTE_FLOW_ITEM_TYPE_IPV6:
1643 flow_verbs_translate_item_ipv6(dev_flow, items,
1645 subpriority = MLX5_PRIORITY_MAP_L3;
1646 dev_flow->hash_fields |=
1647 mlx5_flow_hashfields_adjust
1649 MLX5_IPV6_LAYER_TYPES,
1650 MLX5_IPV6_IBV_RX_HASH);
1651 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1652 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1654 case RTE_FLOW_ITEM_TYPE_TCP:
1655 flow_verbs_translate_item_tcp(dev_flow, items,
1657 subpriority = MLX5_PRIORITY_MAP_L4;
1658 dev_flow->hash_fields |=
1659 mlx5_flow_hashfields_adjust
1660 (dev_flow, tunnel, ETH_RSS_TCP,
1661 (IBV_RX_HASH_SRC_PORT_TCP |
1662 IBV_RX_HASH_DST_PORT_TCP));
1663 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1664 MLX5_FLOW_LAYER_OUTER_L4_TCP;
1666 case RTE_FLOW_ITEM_TYPE_UDP:
1667 flow_verbs_translate_item_udp(dev_flow, items,
1669 subpriority = MLX5_PRIORITY_MAP_L4;
1670 dev_flow->hash_fields |=
1671 mlx5_flow_hashfields_adjust
1672 (dev_flow, tunnel, ETH_RSS_UDP,
1673 (IBV_RX_HASH_SRC_PORT_UDP |
1674 IBV_RX_HASH_DST_PORT_UDP));
1675 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1676 MLX5_FLOW_LAYER_OUTER_L4_UDP;
1678 case RTE_FLOW_ITEM_TYPE_VXLAN:
1679 flow_verbs_translate_item_vxlan(dev_flow, items,
1681 subpriority = MLX5_PRIORITY_MAP_L2;
1682 item_flags |= MLX5_FLOW_LAYER_VXLAN;
1684 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1685 flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1687 subpriority = MLX5_PRIORITY_MAP_L2;
1688 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1690 case RTE_FLOW_ITEM_TYPE_GRE:
1691 flow_verbs_translate_item_gre(dev_flow, items,
1693 subpriority = MLX5_PRIORITY_MAP_L2;
1694 item_flags |= MLX5_FLOW_LAYER_GRE;
1696 case RTE_FLOW_ITEM_TYPE_MPLS:
1697 flow_verbs_translate_item_mpls(dev_flow, items,
1699 subpriority = MLX5_PRIORITY_MAP_L2;
1700 item_flags |= MLX5_FLOW_LAYER_MPLS;
1703 return rte_flow_error_set(error, ENOTSUP,
1704 RTE_FLOW_ERROR_TYPE_ITEM,
1706 "item not supported");
1709 dev_flow->handle->layers = item_flags;
1710 /* Other members of attr will be ignored. */
1711 dev_flow->verbs.attr.priority =
1712 mlx5_flow_adjust_priority(dev, priority, subpriority);
1713 dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port;
1718 * Remove the flow from the NIC but keeps it in memory.
1721 * Pointer to the Ethernet device structure.
1722 * @param[in, out] flow
1723 * Pointer to flow structure.
1726 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1728 struct mlx5_flow_handle *handle;
1732 LIST_FOREACH(handle, &flow->dev_handles, next) {
1733 if (handle->ib_flow) {
1734 claim_zero(mlx5_glue->destroy_flow(handle->ib_flow));
1735 handle->ib_flow = NULL;
1738 if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
1739 mlx5_hrxq_drop_release(dev);
1741 mlx5_hrxq_release(dev, handle->hrxq);
1742 handle->hrxq = NULL;
1744 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1745 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1750 * Remove the flow from the NIC and the memory.
1753 * Pointer to the Ethernet device structure.
1754 * @param[in, out] flow
1755 * Pointer to flow structure.
1758 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1760 struct mlx5_flow_handle *handle;
1764 flow_verbs_remove(dev, flow);
1765 while (!LIST_EMPTY(&flow->dev_handles)) {
1766 handle = LIST_FIRST(&flow->dev_handles);
1767 LIST_REMOVE(handle, next);
1770 if (flow->counter) {
1771 flow_verbs_counter_release(dev, flow->counter);
1777 * Apply the flow to the NIC.
1780 * Pointer to the Ethernet device structure.
1781 * @param[in, out] flow
1782 * Pointer to flow structure.
1784 * Pointer to error structure.
1787 * 0 on success, a negative errno value otherwise and rte_errno is set.
1790 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1791 struct rte_flow_error *error)
1793 struct mlx5_priv *priv = dev->data->dev_private;
1794 struct mlx5_flow_handle *handle;
1795 struct mlx5_flow *dev_flow;
1799 for (idx = priv->flow_idx - 1; idx >= 0; idx--) {
1800 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1801 handle = dev_flow->handle;
1802 if (handle->act_flags & MLX5_FLOW_ACTION_DROP) {
1803 handle->hrxq = mlx5_hrxq_drop_new(dev);
1804 if (!handle->hrxq) {
1807 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1808 "cannot get drop hash queue");
1812 struct mlx5_hrxq *hrxq;
1814 MLX5_ASSERT(flow->rss.queue);
1815 hrxq = mlx5_hrxq_get(dev, flow->rss.key,
1816 MLX5_RSS_HASH_KEY_LEN,
1817 dev_flow->hash_fields,
1819 flow->rss.queue_num);
1821 hrxq = mlx5_hrxq_new(dev, flow->rss.key,
1822 MLX5_RSS_HASH_KEY_LEN,
1823 dev_flow->hash_fields,
1825 flow->rss.queue_num,
1827 MLX5_FLOW_LAYER_TUNNEL));
1831 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1832 "cannot get hash queue");
1835 handle->hrxq = hrxq;
1837 handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp,
1838 &dev_flow->verbs.attr);
1839 if (!handle->ib_flow) {
1840 rte_flow_error_set(error, errno,
1841 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1843 "hardware refuses to create flow");
1846 if (priv->vmwa_context &&
1847 handle->vf_vlan.tag && !handle->vf_vlan.created) {
1849 * The rule contains the VLAN pattern.
1850 * For VF we are going to create VLAN
1851 * interface to make hypervisor set correct
1852 * e-Switch vport context.
1854 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
1859 err = rte_errno; /* Save rte_errno before cleanup. */
1860 LIST_FOREACH(handle, &flow->dev_handles, next) {
1862 if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
1863 mlx5_hrxq_drop_release(dev);
1865 mlx5_hrxq_release(dev, handle->hrxq);
1866 handle->hrxq = NULL;
1868 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1869 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1871 rte_errno = err; /* Restore rte_errno. */
1878 * @see rte_flow_query()
1882 flow_verbs_query(struct rte_eth_dev *dev,
1883 struct rte_flow *flow,
1884 const struct rte_flow_action *actions,
1886 struct rte_flow_error *error)
1890 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1891 switch (actions->type) {
1892 case RTE_FLOW_ACTION_TYPE_VOID:
1894 case RTE_FLOW_ACTION_TYPE_COUNT:
1895 ret = flow_verbs_counter_query(dev, flow, data, error);
1898 return rte_flow_error_set(error, ENOTSUP,
1899 RTE_FLOW_ERROR_TYPE_ACTION,
1901 "action not supported");
1907 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1908 .validate = flow_verbs_validate,
1909 .prepare = flow_verbs_prepare,
1910 .translate = flow_verbs_translate,
1911 .apply = flow_verbs_apply,
1912 .remove = flow_verbs_remove,
1913 .destroy = flow_verbs_destroy,
1914 .query = flow_verbs_query,