1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
31 #include "mlx5_defs.h"
32 #include "mlx5_glue.h"
33 #include "mlx5_flow.h"
35 #include "mlx5_rxtx.h"
37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
43 #ifndef HAVE_MLX5DV_DR_ESWITCH
44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
49 #ifndef HAVE_MLX5DV_DR
50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
53 /* VLAN header definitions */
54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
73 * Initialize flow attributes structure according to flow items' types.
76 * Pointer to item specification.
78 * Pointer to flow attributes structure.
81 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
83 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
85 case RTE_FLOW_ITEM_TYPE_IPV4:
88 case RTE_FLOW_ITEM_TYPE_IPV6:
91 case RTE_FLOW_ITEM_TYPE_UDP:
94 case RTE_FLOW_ITEM_TYPE_TCP:
104 struct field_modify_info {
105 uint32_t size; /* Size of field in protocol header, in bytes. */
106 uint32_t offset; /* Offset of field in protocol header, in bytes. */
107 enum mlx5_modification_field id;
110 struct field_modify_info modify_eth[] = {
111 {4, 0, MLX5_MODI_OUT_DMAC_47_16},
112 {2, 4, MLX5_MODI_OUT_DMAC_15_0},
113 {4, 6, MLX5_MODI_OUT_SMAC_47_16},
114 {2, 10, MLX5_MODI_OUT_SMAC_15_0},
118 struct field_modify_info modify_ipv4[] = {
119 {1, 8, MLX5_MODI_OUT_IPV4_TTL},
120 {4, 12, MLX5_MODI_OUT_SIPV4},
121 {4, 16, MLX5_MODI_OUT_DIPV4},
125 struct field_modify_info modify_ipv6[] = {
126 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
127 {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
128 {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
129 {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
130 {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
131 {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
132 {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
133 {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
134 {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
138 struct field_modify_info modify_udp[] = {
139 {2, 0, MLX5_MODI_OUT_UDP_SPORT},
140 {2, 2, MLX5_MODI_OUT_UDP_DPORT},
144 struct field_modify_info modify_tcp[] = {
145 {2, 0, MLX5_MODI_OUT_TCP_SPORT},
146 {2, 2, MLX5_MODI_OUT_TCP_DPORT},
147 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
148 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
153 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
154 uint8_t next_protocol, uint64_t *item_flags,
157 assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
158 item->type == RTE_FLOW_ITEM_TYPE_IPV6);
159 if (next_protocol == IPPROTO_IPIP) {
160 *item_flags |= MLX5_FLOW_LAYER_IPIP;
163 if (next_protocol == IPPROTO_IPV6) {
164 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
170 * Acquire the synchronizing object to protect multithreaded access
171 * to shared dv context. Lock occurs only if context is actually
172 * shared, i.e. we have multiport IB device and representors are
176 * Pointer to the rte_eth_dev structure.
179 flow_d_shared_lock(struct rte_eth_dev *dev)
181 struct mlx5_priv *priv = dev->data->dev_private;
182 struct mlx5_ibv_shared *sh = priv->sh;
184 if (sh->dv_refcnt > 1) {
187 ret = pthread_mutex_lock(&sh->dv_mutex);
194 flow_d_shared_unlock(struct rte_eth_dev *dev)
196 struct mlx5_priv *priv = dev->data->dev_private;
197 struct mlx5_ibv_shared *sh = priv->sh;
199 if (sh->dv_refcnt > 1) {
202 ret = pthread_mutex_unlock(&sh->dv_mutex);
209 * Convert modify-header action to DV specification.
212 * Pointer to item specification.
214 * Pointer to field modification information.
215 * @param[in,out] resource
216 * Pointer to the modify-header resource.
218 * Type of modification.
220 * Pointer to the error structure.
223 * 0 on success, a negative errno value otherwise and rte_errno is set.
226 flow_dv_convert_modify_action(struct rte_flow_item *item,
227 struct field_modify_info *field,
228 struct mlx5_flow_dv_modify_hdr_resource *resource,
230 struct rte_flow_error *error)
232 uint32_t i = resource->actions_num;
233 struct mlx5_modification_cmd *actions = resource->actions;
234 const uint8_t *spec = item->spec;
235 const uint8_t *mask = item->mask;
238 while (field->size) {
240 /* Generate modify command for each mask segment. */
241 memcpy(&set, &mask[field->offset], field->size);
243 if (i >= MLX5_MODIFY_NUM)
244 return rte_flow_error_set(error, EINVAL,
245 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
246 "too many items to modify");
247 actions[i].action_type = type;
248 actions[i].field = field->id;
249 actions[i].length = field->size ==
250 4 ? 0 : field->size * 8;
251 rte_memcpy(&actions[i].data[4 - field->size],
252 &spec[field->offset], field->size);
253 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
256 if (resource->actions_num != i)
257 resource->actions_num = i;
260 if (!resource->actions_num)
261 return rte_flow_error_set(error, EINVAL,
262 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
263 "invalid modification flow item");
268 * Convert modify-header set IPv4 address action to DV specification.
270 * @param[in,out] resource
271 * Pointer to the modify-header resource.
273 * Pointer to action specification.
275 * Pointer to the error structure.
278 * 0 on success, a negative errno value otherwise and rte_errno is set.
281 flow_dv_convert_action_modify_ipv4
282 (struct mlx5_flow_dv_modify_hdr_resource *resource,
283 const struct rte_flow_action *action,
284 struct rte_flow_error *error)
286 const struct rte_flow_action_set_ipv4 *conf =
287 (const struct rte_flow_action_set_ipv4 *)(action->conf);
288 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
289 struct rte_flow_item_ipv4 ipv4;
290 struct rte_flow_item_ipv4 ipv4_mask;
292 memset(&ipv4, 0, sizeof(ipv4));
293 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
294 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
295 ipv4.hdr.src_addr = conf->ipv4_addr;
296 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
298 ipv4.hdr.dst_addr = conf->ipv4_addr;
299 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
302 item.mask = &ipv4_mask;
303 return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
304 MLX5_MODIFICATION_TYPE_SET, error);
308 * Convert modify-header set IPv6 address action to DV specification.
310 * @param[in,out] resource
311 * Pointer to the modify-header resource.
313 * Pointer to action specification.
315 * Pointer to the error structure.
318 * 0 on success, a negative errno value otherwise and rte_errno is set.
321 flow_dv_convert_action_modify_ipv6
322 (struct mlx5_flow_dv_modify_hdr_resource *resource,
323 const struct rte_flow_action *action,
324 struct rte_flow_error *error)
326 const struct rte_flow_action_set_ipv6 *conf =
327 (const struct rte_flow_action_set_ipv6 *)(action->conf);
328 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
329 struct rte_flow_item_ipv6 ipv6;
330 struct rte_flow_item_ipv6 ipv6_mask;
332 memset(&ipv6, 0, sizeof(ipv6));
333 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
334 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
335 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
336 sizeof(ipv6.hdr.src_addr));
337 memcpy(&ipv6_mask.hdr.src_addr,
338 &rte_flow_item_ipv6_mask.hdr.src_addr,
339 sizeof(ipv6.hdr.src_addr));
341 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
342 sizeof(ipv6.hdr.dst_addr));
343 memcpy(&ipv6_mask.hdr.dst_addr,
344 &rte_flow_item_ipv6_mask.hdr.dst_addr,
345 sizeof(ipv6.hdr.dst_addr));
348 item.mask = &ipv6_mask;
349 return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
350 MLX5_MODIFICATION_TYPE_SET, error);
354 * Convert modify-header set MAC address action to DV specification.
356 * @param[in,out] resource
357 * Pointer to the modify-header resource.
359 * Pointer to action specification.
361 * Pointer to the error structure.
364 * 0 on success, a negative errno value otherwise and rte_errno is set.
367 flow_dv_convert_action_modify_mac
368 (struct mlx5_flow_dv_modify_hdr_resource *resource,
369 const struct rte_flow_action *action,
370 struct rte_flow_error *error)
372 const struct rte_flow_action_set_mac *conf =
373 (const struct rte_flow_action_set_mac *)(action->conf);
374 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
375 struct rte_flow_item_eth eth;
376 struct rte_flow_item_eth eth_mask;
378 memset(ð, 0, sizeof(eth));
379 memset(ð_mask, 0, sizeof(eth_mask));
380 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
381 memcpy(ð.src.addr_bytes, &conf->mac_addr,
382 sizeof(eth.src.addr_bytes));
383 memcpy(ð_mask.src.addr_bytes,
384 &rte_flow_item_eth_mask.src.addr_bytes,
385 sizeof(eth_mask.src.addr_bytes));
387 memcpy(ð.dst.addr_bytes, &conf->mac_addr,
388 sizeof(eth.dst.addr_bytes));
389 memcpy(ð_mask.dst.addr_bytes,
390 &rte_flow_item_eth_mask.dst.addr_bytes,
391 sizeof(eth_mask.dst.addr_bytes));
394 item.mask = ð_mask;
395 return flow_dv_convert_modify_action(&item, modify_eth, resource,
396 MLX5_MODIFICATION_TYPE_SET, error);
400 * Convert modify-header set TP action to DV specification.
402 * @param[in,out] resource
403 * Pointer to the modify-header resource.
405 * Pointer to action specification.
407 * Pointer to rte_flow_item objects list.
409 * Pointer to flow attributes structure.
411 * Pointer to the error structure.
414 * 0 on success, a negative errno value otherwise and rte_errno is set.
417 flow_dv_convert_action_modify_tp
418 (struct mlx5_flow_dv_modify_hdr_resource *resource,
419 const struct rte_flow_action *action,
420 const struct rte_flow_item *items,
421 union flow_dv_attr *attr,
422 struct rte_flow_error *error)
424 const struct rte_flow_action_set_tp *conf =
425 (const struct rte_flow_action_set_tp *)(action->conf);
426 struct rte_flow_item item;
427 struct rte_flow_item_udp udp;
428 struct rte_flow_item_udp udp_mask;
429 struct rte_flow_item_tcp tcp;
430 struct rte_flow_item_tcp tcp_mask;
431 struct field_modify_info *field;
434 flow_dv_attr_init(items, attr);
436 memset(&udp, 0, sizeof(udp));
437 memset(&udp_mask, 0, sizeof(udp_mask));
438 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
439 udp.hdr.src_port = conf->port;
440 udp_mask.hdr.src_port =
441 rte_flow_item_udp_mask.hdr.src_port;
443 udp.hdr.dst_port = conf->port;
444 udp_mask.hdr.dst_port =
445 rte_flow_item_udp_mask.hdr.dst_port;
447 item.type = RTE_FLOW_ITEM_TYPE_UDP;
449 item.mask = &udp_mask;
453 memset(&tcp, 0, sizeof(tcp));
454 memset(&tcp_mask, 0, sizeof(tcp_mask));
455 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
456 tcp.hdr.src_port = conf->port;
457 tcp_mask.hdr.src_port =
458 rte_flow_item_tcp_mask.hdr.src_port;
460 tcp.hdr.dst_port = conf->port;
461 tcp_mask.hdr.dst_port =
462 rte_flow_item_tcp_mask.hdr.dst_port;
464 item.type = RTE_FLOW_ITEM_TYPE_TCP;
466 item.mask = &tcp_mask;
469 return flow_dv_convert_modify_action(&item, field, resource,
470 MLX5_MODIFICATION_TYPE_SET, error);
474 * Convert modify-header set TTL action to DV specification.
476 * @param[in,out] resource
477 * Pointer to the modify-header resource.
479 * Pointer to action specification.
481 * Pointer to rte_flow_item objects list.
483 * Pointer to flow attributes structure.
485 * Pointer to the error structure.
488 * 0 on success, a negative errno value otherwise and rte_errno is set.
491 flow_dv_convert_action_modify_ttl
492 (struct mlx5_flow_dv_modify_hdr_resource *resource,
493 const struct rte_flow_action *action,
494 const struct rte_flow_item *items,
495 union flow_dv_attr *attr,
496 struct rte_flow_error *error)
498 const struct rte_flow_action_set_ttl *conf =
499 (const struct rte_flow_action_set_ttl *)(action->conf);
500 struct rte_flow_item item;
501 struct rte_flow_item_ipv4 ipv4;
502 struct rte_flow_item_ipv4 ipv4_mask;
503 struct rte_flow_item_ipv6 ipv6;
504 struct rte_flow_item_ipv6 ipv6_mask;
505 struct field_modify_info *field;
508 flow_dv_attr_init(items, attr);
510 memset(&ipv4, 0, sizeof(ipv4));
511 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
512 ipv4.hdr.time_to_live = conf->ttl_value;
513 ipv4_mask.hdr.time_to_live = 0xFF;
514 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
516 item.mask = &ipv4_mask;
520 memset(&ipv6, 0, sizeof(ipv6));
521 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
522 ipv6.hdr.hop_limits = conf->ttl_value;
523 ipv6_mask.hdr.hop_limits = 0xFF;
524 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
526 item.mask = &ipv6_mask;
529 return flow_dv_convert_modify_action(&item, field, resource,
530 MLX5_MODIFICATION_TYPE_SET, error);
534 * Convert modify-header decrement TTL action to DV specification.
536 * @param[in,out] resource
537 * Pointer to the modify-header resource.
539 * Pointer to action specification.
541 * Pointer to rte_flow_item objects list.
543 * Pointer to flow attributes structure.
545 * Pointer to the error structure.
548 * 0 on success, a negative errno value otherwise and rte_errno is set.
551 flow_dv_convert_action_modify_dec_ttl
552 (struct mlx5_flow_dv_modify_hdr_resource *resource,
553 const struct rte_flow_item *items,
554 union flow_dv_attr *attr,
555 struct rte_flow_error *error)
557 struct rte_flow_item item;
558 struct rte_flow_item_ipv4 ipv4;
559 struct rte_flow_item_ipv4 ipv4_mask;
560 struct rte_flow_item_ipv6 ipv6;
561 struct rte_flow_item_ipv6 ipv6_mask;
562 struct field_modify_info *field;
565 flow_dv_attr_init(items, attr);
567 memset(&ipv4, 0, sizeof(ipv4));
568 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
569 ipv4.hdr.time_to_live = 0xFF;
570 ipv4_mask.hdr.time_to_live = 0xFF;
571 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
573 item.mask = &ipv4_mask;
577 memset(&ipv6, 0, sizeof(ipv6));
578 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
579 ipv6.hdr.hop_limits = 0xFF;
580 ipv6_mask.hdr.hop_limits = 0xFF;
581 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
583 item.mask = &ipv6_mask;
586 return flow_dv_convert_modify_action(&item, field, resource,
587 MLX5_MODIFICATION_TYPE_ADD, error);
591 * Convert modify-header increment/decrement TCP Sequence number
592 * to DV specification.
594 * @param[in,out] resource
595 * Pointer to the modify-header resource.
597 * Pointer to action specification.
599 * Pointer to the error structure.
602 * 0 on success, a negative errno value otherwise and rte_errno is set.
605 flow_dv_convert_action_modify_tcp_seq
606 (struct mlx5_flow_dv_modify_hdr_resource *resource,
607 const struct rte_flow_action *action,
608 struct rte_flow_error *error)
610 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
611 uint64_t value = rte_be_to_cpu_32(*conf);
612 struct rte_flow_item item;
613 struct rte_flow_item_tcp tcp;
614 struct rte_flow_item_tcp tcp_mask;
616 memset(&tcp, 0, sizeof(tcp));
617 memset(&tcp_mask, 0, sizeof(tcp_mask));
618 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
620 * The HW has no decrement operation, only increment operation.
621 * To simulate decrement X from Y using increment operation
622 * we need to add UINT32_MAX X times to Y.
623 * Each adding of UINT32_MAX decrements Y by 1.
626 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
627 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
628 item.type = RTE_FLOW_ITEM_TYPE_TCP;
630 item.mask = &tcp_mask;
631 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
632 MLX5_MODIFICATION_TYPE_ADD, error);
636 * Convert modify-header increment/decrement TCP Acknowledgment number
637 * to DV specification.
639 * @param[in,out] resource
640 * Pointer to the modify-header resource.
642 * Pointer to action specification.
644 * Pointer to the error structure.
647 * 0 on success, a negative errno value otherwise and rte_errno is set.
650 flow_dv_convert_action_modify_tcp_ack
651 (struct mlx5_flow_dv_modify_hdr_resource *resource,
652 const struct rte_flow_action *action,
653 struct rte_flow_error *error)
655 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
656 uint64_t value = rte_be_to_cpu_32(*conf);
657 struct rte_flow_item item;
658 struct rte_flow_item_tcp tcp;
659 struct rte_flow_item_tcp tcp_mask;
661 memset(&tcp, 0, sizeof(tcp));
662 memset(&tcp_mask, 0, sizeof(tcp_mask));
663 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
665 * The HW has no decrement operation, only increment operation.
666 * To simulate decrement X from Y using increment operation
667 * we need to add UINT32_MAX X times to Y.
668 * Each adding of UINT32_MAX decrements Y by 1.
671 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
672 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
673 item.type = RTE_FLOW_ITEM_TYPE_TCP;
675 item.mask = &tcp_mask;
676 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
677 MLX5_MODIFICATION_TYPE_ADD, error);
681 * Validate META item.
684 * Pointer to the rte_eth_dev structure.
686 * Item specification.
688 * Attributes of flow that includes this item.
690 * Pointer to error structure.
693 * 0 on success, a negative errno value otherwise and rte_errno is set.
696 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
697 const struct rte_flow_item *item,
698 const struct rte_flow_attr *attr,
699 struct rte_flow_error *error)
701 const struct rte_flow_item_meta *spec = item->spec;
702 const struct rte_flow_item_meta *mask = item->mask;
703 const struct rte_flow_item_meta nic_mask = {
704 .data = RTE_BE32(UINT32_MAX)
707 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
709 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
710 return rte_flow_error_set(error, EPERM,
711 RTE_FLOW_ERROR_TYPE_ITEM,
713 "match on metadata offload "
714 "configuration is off for this port");
716 return rte_flow_error_set(error, EINVAL,
717 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
719 "data cannot be empty");
721 return rte_flow_error_set(error, EINVAL,
722 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
724 "data cannot be zero");
726 mask = &rte_flow_item_meta_mask;
727 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
728 (const uint8_t *)&nic_mask,
729 sizeof(struct rte_flow_item_meta),
734 return rte_flow_error_set(error, ENOTSUP,
735 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
737 "pattern not supported for ingress");
742 * Validate vport item.
745 * Pointer to the rte_eth_dev structure.
747 * Item specification.
749 * Attributes of flow that includes this item.
750 * @param[in] item_flags
751 * Bit-fields that holds the items detected until now.
753 * Pointer to error structure.
756 * 0 on success, a negative errno value otherwise and rte_errno is set.
759 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
760 const struct rte_flow_item *item,
761 const struct rte_flow_attr *attr,
763 struct rte_flow_error *error)
765 const struct rte_flow_item_port_id *spec = item->spec;
766 const struct rte_flow_item_port_id *mask = item->mask;
767 const struct rte_flow_item_port_id switch_mask = {
770 uint16_t esw_domain_id;
771 uint16_t item_port_esw_domain_id;
775 return rte_flow_error_set(error, EINVAL,
776 RTE_FLOW_ERROR_TYPE_ITEM,
778 "match on port id is valid only"
779 " when transfer flag is enabled");
780 if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
781 return rte_flow_error_set(error, ENOTSUP,
782 RTE_FLOW_ERROR_TYPE_ITEM, item,
783 "multiple source ports are not"
787 if (mask->id != 0xffffffff)
788 return rte_flow_error_set(error, ENOTSUP,
789 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
791 "no support for partial mask on"
793 ret = mlx5_flow_item_acceptable
794 (item, (const uint8_t *)mask,
795 (const uint8_t *)&rte_flow_item_port_id_mask,
796 sizeof(struct rte_flow_item_port_id),
802 ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
805 return rte_flow_error_set(error, -ret,
806 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
807 "failed to obtain E-Switch info for"
809 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
810 &esw_domain_id, NULL);
812 return rte_flow_error_set(error, -ret,
813 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
815 "failed to obtain E-Switch info");
816 if (item_port_esw_domain_id != esw_domain_id)
817 return rte_flow_error_set(error, -ret,
818 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
819 "cannot match on a port from a"
820 " different E-Switch");
825 * Validate the pop VLAN action.
828 * Pointer to the rte_eth_dev structure.
829 * @param[in] action_flags
830 * Holds the actions detected until now.
832 * Pointer to the pop vlan action.
833 * @param[in] item_flags
834 * The items found in this flow rule.
836 * Pointer to flow attributes.
838 * Pointer to error structure.
841 * 0 on success, a negative errno value otherwise and rte_errno is set.
844 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
845 uint64_t action_flags,
846 const struct rte_flow_action *action,
848 const struct rte_flow_attr *attr,
849 struct rte_flow_error *error)
851 struct mlx5_priv *priv = dev->data->dev_private;
855 if (!priv->sh->pop_vlan_action)
856 return rte_flow_error_set(error, ENOTSUP,
857 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
859 "pop vlan action is not supported");
861 * Check for inconsistencies:
862 * fail strip_vlan in a flow that matches packets without VLAN tags.
863 * fail strip_vlan in a flow that matches packets without explicitly a
864 * matching on VLAN tag ?
866 if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN)
867 return rte_flow_error_set(error, ENOTSUP,
868 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
870 "no support for multiple vlan pop "
872 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
873 return rte_flow_error_set(error, ENOTSUP,
874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
876 "cannot pop vlan without a "
877 "match on (outer) vlan in the flow");
882 * Get VLAN default info from vlan match info.
885 * Pointer to the rte_eth_dev structure.
887 * the list of item specifications.
889 * pointer VLAN info to fill to.
891 * Pointer to error structure.
894 * 0 on success, a negative errno value otherwise and rte_errno is set.
897 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
898 struct rte_vlan_hdr *vlan)
900 const struct rte_flow_item_vlan nic_mask = {
901 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
902 MLX5DV_FLOW_VLAN_VID_MASK),
903 .inner_type = RTE_BE16(0xffff),
908 for (; items->type != RTE_FLOW_ITEM_TYPE_END &&
909 items->type != RTE_FLOW_ITEM_TYPE_VLAN; items++)
911 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
912 const struct rte_flow_item_vlan *vlan_m = items->mask;
913 const struct rte_flow_item_vlan *vlan_v = items->spec;
917 /* Only full match values are accepted */
918 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
919 MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
920 vlan->vlan_tci &= MLX5DV_FLOW_VLAN_PCP_MASK;
922 rte_be_to_cpu_16(vlan_v->tci &
923 MLX5DV_FLOW_VLAN_PCP_MASK_BE);
925 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
926 MLX5DV_FLOW_VLAN_VID_MASK_BE) {
927 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
929 rte_be_to_cpu_16(vlan_v->tci &
930 MLX5DV_FLOW_VLAN_VID_MASK_BE);
932 if (vlan_m->inner_type == nic_mask.inner_type)
933 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
939 * Validate the push VLAN action.
941 * @param[in] action_flags
942 * Holds the actions detected until now.
944 * Pointer to the encap action.
946 * Pointer to flow attributes
948 * Pointer to error structure.
951 * 0 on success, a negative errno value otherwise and rte_errno is set.
954 flow_dv_validate_action_push_vlan(uint64_t action_flags,
955 const struct rte_flow_action *action,
956 const struct rte_flow_attr *attr,
957 struct rte_flow_error *error)
959 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
961 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
962 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
963 return rte_flow_error_set(error, EINVAL,
964 RTE_FLOW_ERROR_TYPE_ACTION, action,
965 "invalid vlan ethertype");
967 (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN))
968 return rte_flow_error_set(error, ENOTSUP,
969 RTE_FLOW_ERROR_TYPE_ACTION, action,
970 "no support for multiple VLAN "
977 * Validate the set VLAN PCP.
979 * @param[in] action_flags
980 * Holds the actions detected until now.
982 * Pointer to the list of actions remaining in the flow rule.
984 * Pointer to flow attributes
986 * Pointer to error structure.
989 * 0 on success, a negative errno value otherwise and rte_errno is set.
992 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
993 const struct rte_flow_action actions[],
994 struct rte_flow_error *error)
996 const struct rte_flow_action *action = actions;
997 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
999 if (conf->vlan_pcp > 7)
1000 return rte_flow_error_set(error, EINVAL,
1001 RTE_FLOW_ERROR_TYPE_ACTION, action,
1002 "VLAN PCP value is too big");
1003 if (mlx5_flow_find_action(actions,
1004 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) == NULL)
1005 return rte_flow_error_set(error, ENOTSUP,
1006 RTE_FLOW_ERROR_TYPE_ACTION, action,
1007 "set VLAN PCP can only be used "
1008 "with push VLAN action");
1009 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
1010 return rte_flow_error_set(error, ENOTSUP,
1011 RTE_FLOW_ERROR_TYPE_ACTION, action,
1012 "set VLAN PCP action must precede "
1013 "the push VLAN action");
1018 * Validate count action.
1023 * Pointer to error structure.
1026 * 0 on success, a negative errno value otherwise and rte_errno is set.
1029 flow_dv_validate_action_count(struct rte_eth_dev *dev,
1030 struct rte_flow_error *error)
1032 struct mlx5_priv *priv = dev->data->dev_private;
1034 if (!priv->config.devx)
1036 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
1040 return rte_flow_error_set
1042 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1044 "count action not supported");
1048 * Validate the L2 encap action.
1050 * @param[in] action_flags
1051 * Holds the actions detected until now.
1053 * Pointer to the encap action.
1055 * Pointer to flow attributes
1057 * Pointer to error structure.
1060 * 0 on success, a negative errno value otherwise and rte_errno is set.
1063 flow_dv_validate_action_l2_encap(uint64_t action_flags,
1064 const struct rte_flow_action *action,
1065 const struct rte_flow_attr *attr,
1066 struct rte_flow_error *error)
1068 if (!(action->conf))
1069 return rte_flow_error_set(error, EINVAL,
1070 RTE_FLOW_ERROR_TYPE_ACTION, action,
1071 "configuration cannot be null");
1072 if (action_flags & MLX5_FLOW_ACTION_DROP)
1073 return rte_flow_error_set(error, EINVAL,
1074 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1075 "can't drop and encap in same flow");
1076 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1077 return rte_flow_error_set(error, EINVAL,
1078 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1079 "can only have a single encap or"
1080 " decap action in a flow");
1081 if (!attr->transfer && attr->ingress)
1082 return rte_flow_error_set(error, ENOTSUP,
1083 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1085 "encap action not supported for "
1091 * Validate the L2 decap action.
1093 * @param[in] action_flags
1094 * Holds the actions detected until now.
1096 * Pointer to flow attributes
1098 * Pointer to error structure.
1101 * 0 on success, a negative errno value otherwise and rte_errno is set.
1104 flow_dv_validate_action_l2_decap(uint64_t action_flags,
1105 const struct rte_flow_attr *attr,
1106 struct rte_flow_error *error)
1108 if (action_flags & MLX5_FLOW_ACTION_DROP)
1109 return rte_flow_error_set(error, EINVAL,
1110 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1111 "can't drop and decap in same flow");
1112 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1113 return rte_flow_error_set(error, EINVAL,
1114 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1115 "can only have a single encap or"
1116 " decap action in a flow");
1117 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1118 return rte_flow_error_set(error, EINVAL,
1119 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1120 "can't have decap action after"
1123 return rte_flow_error_set(error, ENOTSUP,
1124 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1126 "decap action not supported for "
1132 * Validate the raw encap action.
1134 * @param[in] action_flags
1135 * Holds the actions detected until now.
1137 * Pointer to the encap action.
1139 * Pointer to flow attributes
1141 * Pointer to error structure.
1144 * 0 on success, a negative errno value otherwise and rte_errno is set.
1147 flow_dv_validate_action_raw_encap(uint64_t action_flags,
1148 const struct rte_flow_action *action,
1149 const struct rte_flow_attr *attr,
1150 struct rte_flow_error *error)
1152 const struct rte_flow_action_raw_encap *raw_encap =
1153 (const struct rte_flow_action_raw_encap *)action->conf;
1154 if (!(action->conf))
1155 return rte_flow_error_set(error, EINVAL,
1156 RTE_FLOW_ERROR_TYPE_ACTION, action,
1157 "configuration cannot be null");
1158 if (action_flags & MLX5_FLOW_ACTION_DROP)
1159 return rte_flow_error_set(error, EINVAL,
1160 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1161 "can't drop and encap in same flow");
1162 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1163 return rte_flow_error_set(error, EINVAL,
1164 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1165 "can only have a single encap"
1166 " action in a flow");
1167 /* encap without preceding decap is not supported for ingress */
1168 if (!attr->transfer && attr->ingress &&
1169 !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
1170 return rte_flow_error_set(error, ENOTSUP,
1171 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1173 "encap action not supported for "
1175 if (!raw_encap->size || !raw_encap->data)
1176 return rte_flow_error_set(error, EINVAL,
1177 RTE_FLOW_ERROR_TYPE_ACTION, action,
1178 "raw encap data cannot be empty");
1183 * Validate the raw decap action.
1185 * @param[in] action_flags
1186 * Holds the actions detected until now.
1188 * Pointer to the encap action.
1190 * Pointer to flow attributes
1192 * Pointer to error structure.
1195 * 0 on success, a negative errno value otherwise and rte_errno is set.
1198 flow_dv_validate_action_raw_decap(uint64_t action_flags,
1199 const struct rte_flow_action *action,
1200 const struct rte_flow_attr *attr,
1201 struct rte_flow_error *error)
1203 if (action_flags & MLX5_FLOW_ACTION_DROP)
1204 return rte_flow_error_set(error, EINVAL,
1205 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1206 "can't drop and decap in same flow");
1207 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1208 return rte_flow_error_set(error, EINVAL,
1209 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1210 "can't have encap action before"
1212 if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1213 return rte_flow_error_set(error, EINVAL,
1214 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1215 "can only have a single decap"
1216 " action in a flow");
1217 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1218 return rte_flow_error_set(error, EINVAL,
1219 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1220 "can't have decap action after"
1222 /* decap action is valid on egress only if it is followed by encap */
1224 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1225 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1228 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1229 return rte_flow_error_set
1231 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1232 NULL, "decap action not supported"
1239 * Find existing encap/decap resource or create and register a new one.
1241 * @param dev[in, out]
1242 * Pointer to rte_eth_dev structure.
1243 * @param[in, out] resource
1244 * Pointer to encap/decap resource.
1245 * @parm[in, out] dev_flow
1246 * Pointer to the dev_flow.
1248 * pointer to error structure.
1251 * 0 on success otherwise -errno and errno is set.
1254 flow_dv_encap_decap_resource_register
1255 (struct rte_eth_dev *dev,
1256 struct mlx5_flow_dv_encap_decap_resource *resource,
1257 struct mlx5_flow *dev_flow,
1258 struct rte_flow_error *error)
1260 struct mlx5_priv *priv = dev->data->dev_private;
1261 struct mlx5_ibv_shared *sh = priv->sh;
1262 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1263 struct rte_flow *flow = dev_flow->flow;
1264 struct mlx5dv_dr_domain *domain;
1266 resource->flags = flow->group ? 0 : 1;
1267 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1268 domain = sh->fdb_domain;
1269 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1270 domain = sh->rx_domain;
1272 domain = sh->tx_domain;
1274 /* Lookup a matching resource from cache. */
1275 LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1276 if (resource->reformat_type == cache_resource->reformat_type &&
1277 resource->ft_type == cache_resource->ft_type &&
1278 resource->flags == cache_resource->flags &&
1279 resource->size == cache_resource->size &&
1280 !memcmp((const void *)resource->buf,
1281 (const void *)cache_resource->buf,
1283 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1284 (void *)cache_resource,
1285 rte_atomic32_read(&cache_resource->refcnt));
1286 rte_atomic32_inc(&cache_resource->refcnt);
1287 dev_flow->dv.encap_decap = cache_resource;
1291 /* Register new encap/decap resource. */
1292 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1293 if (!cache_resource)
1294 return rte_flow_error_set(error, ENOMEM,
1295 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1296 "cannot allocate resource memory");
1297 *cache_resource = *resource;
1298 cache_resource->verbs_action =
1299 mlx5_glue->dv_create_flow_action_packet_reformat
1300 (sh->ctx, cache_resource->reformat_type,
1301 cache_resource->ft_type, domain, cache_resource->flags,
1302 cache_resource->size,
1303 (cache_resource->size ? cache_resource->buf : NULL));
1304 if (!cache_resource->verbs_action) {
1305 rte_free(cache_resource);
1306 return rte_flow_error_set(error, ENOMEM,
1307 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1308 NULL, "cannot create action");
1310 rte_atomic32_init(&cache_resource->refcnt);
1311 rte_atomic32_inc(&cache_resource->refcnt);
1312 LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1313 dev_flow->dv.encap_decap = cache_resource;
1314 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1315 (void *)cache_resource,
1316 rte_atomic32_read(&cache_resource->refcnt));
1321 * Find existing table jump resource or create and register a new one.
1323 * @param dev[in, out]
1324 * Pointer to rte_eth_dev structure.
1325 * @param[in, out] resource
1326 * Pointer to jump table resource.
1327 * @parm[in, out] dev_flow
1328 * Pointer to the dev_flow.
1330 * pointer to error structure.
1333 * 0 on success otherwise -errno and errno is set.
1336 flow_dv_jump_tbl_resource_register
1337 (struct rte_eth_dev *dev,
1338 struct mlx5_flow_dv_jump_tbl_resource *resource,
1339 struct mlx5_flow *dev_flow,
1340 struct rte_flow_error *error)
1342 struct mlx5_priv *priv = dev->data->dev_private;
1343 struct mlx5_ibv_shared *sh = priv->sh;
1344 struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1346 /* Lookup a matching resource from cache. */
1347 LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1348 if (resource->tbl == cache_resource->tbl) {
1349 DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1350 (void *)cache_resource,
1351 rte_atomic32_read(&cache_resource->refcnt));
1352 rte_atomic32_inc(&cache_resource->refcnt);
1353 dev_flow->dv.jump = cache_resource;
1357 /* Register new jump table resource. */
1358 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1359 if (!cache_resource)
1360 return rte_flow_error_set(error, ENOMEM,
1361 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1362 "cannot allocate resource memory");
1363 *cache_resource = *resource;
1364 cache_resource->action =
1365 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1366 (resource->tbl->obj);
1367 if (!cache_resource->action) {
1368 rte_free(cache_resource);
1369 return rte_flow_error_set(error, ENOMEM,
1370 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1371 NULL, "cannot create action");
1373 rte_atomic32_init(&cache_resource->refcnt);
1374 rte_atomic32_inc(&cache_resource->refcnt);
1375 LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1376 dev_flow->dv.jump = cache_resource;
1377 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
1378 (void *)cache_resource,
1379 rte_atomic32_read(&cache_resource->refcnt));
1384 * Find existing table port ID resource or create and register a new one.
1386 * @param dev[in, out]
1387 * Pointer to rte_eth_dev structure.
1388 * @param[in, out] resource
1389 * Pointer to port ID action resource.
1390 * @parm[in, out] dev_flow
1391 * Pointer to the dev_flow.
1393 * pointer to error structure.
1396 * 0 on success otherwise -errno and errno is set.
1399 flow_dv_port_id_action_resource_register
1400 (struct rte_eth_dev *dev,
1401 struct mlx5_flow_dv_port_id_action_resource *resource,
1402 struct mlx5_flow *dev_flow,
1403 struct rte_flow_error *error)
1405 struct mlx5_priv *priv = dev->data->dev_private;
1406 struct mlx5_ibv_shared *sh = priv->sh;
1407 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1409 /* Lookup a matching resource from cache. */
1410 LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1411 if (resource->port_id == cache_resource->port_id) {
1412 DRV_LOG(DEBUG, "port id action resource resource %p: "
1414 (void *)cache_resource,
1415 rte_atomic32_read(&cache_resource->refcnt));
1416 rte_atomic32_inc(&cache_resource->refcnt);
1417 dev_flow->dv.port_id_action = cache_resource;
1421 /* Register new port id action resource. */
1422 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1423 if (!cache_resource)
1424 return rte_flow_error_set(error, ENOMEM,
1425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1426 "cannot allocate resource memory");
1427 *cache_resource = *resource;
1428 cache_resource->action =
1429 mlx5_glue->dr_create_flow_action_dest_vport
1430 (priv->sh->fdb_domain, resource->port_id);
1431 if (!cache_resource->action) {
1432 rte_free(cache_resource);
1433 return rte_flow_error_set(error, ENOMEM,
1434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1435 NULL, "cannot create action");
1437 rte_atomic32_init(&cache_resource->refcnt);
1438 rte_atomic32_inc(&cache_resource->refcnt);
1439 LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1440 dev_flow->dv.port_id_action = cache_resource;
1441 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1442 (void *)cache_resource,
1443 rte_atomic32_read(&cache_resource->refcnt));
1448 * Find existing push vlan resource or create and register a new one.
1450 * @param dev[in, out]
1451 * Pointer to rte_eth_dev structure.
1452 * @param[in, out] resource
1453 * Pointer to port ID action resource.
1454 * @parm[in, out] dev_flow
1455 * Pointer to the dev_flow.
1457 * pointer to error structure.
1460 * 0 on success otherwise -errno and errno is set.
1463 flow_dv_push_vlan_action_resource_register
1464 (struct rte_eth_dev *dev,
1465 struct mlx5_flow_dv_push_vlan_action_resource *resource,
1466 struct mlx5_flow *dev_flow,
1467 struct rte_flow_error *error)
1469 struct mlx5_priv *priv = dev->data->dev_private;
1470 struct mlx5_ibv_shared *sh = priv->sh;
1471 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
1472 struct mlx5dv_dr_domain *domain;
1474 /* Lookup a matching resource from cache. */
1475 LIST_FOREACH(cache_resource, &sh->push_vlan_action_list, next) {
1476 if (resource->vlan_tag == cache_resource->vlan_tag &&
1477 resource->ft_type == cache_resource->ft_type) {
1478 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
1480 (void *)cache_resource,
1481 rte_atomic32_read(&cache_resource->refcnt));
1482 rte_atomic32_inc(&cache_resource->refcnt);
1483 dev_flow->dv.push_vlan_res = cache_resource;
1487 /* Register new push_vlan action resource. */
1488 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1489 if (!cache_resource)
1490 return rte_flow_error_set(error, ENOMEM,
1491 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1492 "cannot allocate resource memory");
1493 *cache_resource = *resource;
1494 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1495 domain = sh->fdb_domain;
1496 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1497 domain = sh->rx_domain;
1499 domain = sh->tx_domain;
1500 cache_resource->action =
1501 mlx5_glue->dr_create_flow_action_push_vlan(domain,
1502 resource->vlan_tag);
1503 if (!cache_resource->action) {
1504 rte_free(cache_resource);
1505 return rte_flow_error_set(error, ENOMEM,
1506 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1507 NULL, "cannot create action");
1509 rte_atomic32_init(&cache_resource->refcnt);
1510 rte_atomic32_inc(&cache_resource->refcnt);
1511 LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next);
1512 dev_flow->dv.push_vlan_res = cache_resource;
1513 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
1514 (void *)cache_resource,
1515 rte_atomic32_read(&cache_resource->refcnt));
1519 * Get the size of specific rte_flow_item_type
1521 * @param[in] item_type
1522 * Tested rte_flow_item_type.
1525 * sizeof struct item_type, 0 if void or irrelevant.
1528 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1532 switch (item_type) {
1533 case RTE_FLOW_ITEM_TYPE_ETH:
1534 retval = sizeof(struct rte_flow_item_eth);
1536 case RTE_FLOW_ITEM_TYPE_VLAN:
1537 retval = sizeof(struct rte_flow_item_vlan);
1539 case RTE_FLOW_ITEM_TYPE_IPV4:
1540 retval = sizeof(struct rte_flow_item_ipv4);
1542 case RTE_FLOW_ITEM_TYPE_IPV6:
1543 retval = sizeof(struct rte_flow_item_ipv6);
1545 case RTE_FLOW_ITEM_TYPE_UDP:
1546 retval = sizeof(struct rte_flow_item_udp);
1548 case RTE_FLOW_ITEM_TYPE_TCP:
1549 retval = sizeof(struct rte_flow_item_tcp);
1551 case RTE_FLOW_ITEM_TYPE_VXLAN:
1552 retval = sizeof(struct rte_flow_item_vxlan);
1554 case RTE_FLOW_ITEM_TYPE_GRE:
1555 retval = sizeof(struct rte_flow_item_gre);
1557 case RTE_FLOW_ITEM_TYPE_NVGRE:
1558 retval = sizeof(struct rte_flow_item_nvgre);
1560 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1561 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1563 case RTE_FLOW_ITEM_TYPE_MPLS:
1564 retval = sizeof(struct rte_flow_item_mpls);
1566 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1574 #define MLX5_ENCAP_IPV4_VERSION 0x40
1575 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
1576 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
1577 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
1578 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
1579 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
1580 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
1583 * Convert the encap action data from list of rte_flow_item to raw buffer
1586 * Pointer to rte_flow_item objects list.
1588 * Pointer to the output buffer.
1590 * Pointer to the output buffer size.
1592 * Pointer to the error structure.
1595 * 0 on success, a negative errno value otherwise and rte_errno is set.
1598 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1599 size_t *size, struct rte_flow_error *error)
1601 struct rte_ether_hdr *eth = NULL;
1602 struct rte_vlan_hdr *vlan = NULL;
1603 struct rte_ipv4_hdr *ipv4 = NULL;
1604 struct rte_ipv6_hdr *ipv6 = NULL;
1605 struct rte_udp_hdr *udp = NULL;
1606 struct rte_vxlan_hdr *vxlan = NULL;
1607 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1608 struct rte_gre_hdr *gre = NULL;
1610 size_t temp_size = 0;
1613 return rte_flow_error_set(error, EINVAL,
1614 RTE_FLOW_ERROR_TYPE_ACTION,
1615 NULL, "invalid empty data");
1616 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1617 len = flow_dv_get_item_len(items->type);
1618 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1619 return rte_flow_error_set(error, EINVAL,
1620 RTE_FLOW_ERROR_TYPE_ACTION,
1621 (void *)items->type,
1622 "items total size is too big"
1623 " for encap action");
1624 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1625 switch (items->type) {
1626 case RTE_FLOW_ITEM_TYPE_ETH:
1627 eth = (struct rte_ether_hdr *)&buf[temp_size];
1629 case RTE_FLOW_ITEM_TYPE_VLAN:
1630 vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1632 return rte_flow_error_set(error, EINVAL,
1633 RTE_FLOW_ERROR_TYPE_ACTION,
1634 (void *)items->type,
1635 "eth header not found");
1636 if (!eth->ether_type)
1637 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1639 case RTE_FLOW_ITEM_TYPE_IPV4:
1640 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1642 return rte_flow_error_set(error, EINVAL,
1643 RTE_FLOW_ERROR_TYPE_ACTION,
1644 (void *)items->type,
1645 "neither eth nor vlan"
1647 if (vlan && !vlan->eth_proto)
1648 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1649 else if (eth && !eth->ether_type)
1650 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1651 if (!ipv4->version_ihl)
1652 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1653 MLX5_ENCAP_IPV4_IHL_MIN;
1654 if (!ipv4->time_to_live)
1655 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1657 case RTE_FLOW_ITEM_TYPE_IPV6:
1658 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1660 return rte_flow_error_set(error, EINVAL,
1661 RTE_FLOW_ERROR_TYPE_ACTION,
1662 (void *)items->type,
1663 "neither eth nor vlan"
1665 if (vlan && !vlan->eth_proto)
1666 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1667 else if (eth && !eth->ether_type)
1668 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1669 if (!ipv6->vtc_flow)
1671 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1672 if (!ipv6->hop_limits)
1673 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1675 case RTE_FLOW_ITEM_TYPE_UDP:
1676 udp = (struct rte_udp_hdr *)&buf[temp_size];
1678 return rte_flow_error_set(error, EINVAL,
1679 RTE_FLOW_ERROR_TYPE_ACTION,
1680 (void *)items->type,
1681 "ip header not found");
1682 if (ipv4 && !ipv4->next_proto_id)
1683 ipv4->next_proto_id = IPPROTO_UDP;
1684 else if (ipv6 && !ipv6->proto)
1685 ipv6->proto = IPPROTO_UDP;
1687 case RTE_FLOW_ITEM_TYPE_VXLAN:
1688 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1690 return rte_flow_error_set(error, EINVAL,
1691 RTE_FLOW_ERROR_TYPE_ACTION,
1692 (void *)items->type,
1693 "udp header not found");
1695 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1696 if (!vxlan->vx_flags)
1698 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1700 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1701 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1703 return rte_flow_error_set(error, EINVAL,
1704 RTE_FLOW_ERROR_TYPE_ACTION,
1705 (void *)items->type,
1706 "udp header not found");
1707 if (!vxlan_gpe->proto)
1708 return rte_flow_error_set(error, EINVAL,
1709 RTE_FLOW_ERROR_TYPE_ACTION,
1710 (void *)items->type,
1711 "next protocol not found");
1714 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1715 if (!vxlan_gpe->vx_flags)
1716 vxlan_gpe->vx_flags =
1717 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1719 case RTE_FLOW_ITEM_TYPE_GRE:
1720 case RTE_FLOW_ITEM_TYPE_NVGRE:
1721 gre = (struct rte_gre_hdr *)&buf[temp_size];
1723 return rte_flow_error_set(error, EINVAL,
1724 RTE_FLOW_ERROR_TYPE_ACTION,
1725 (void *)items->type,
1726 "next protocol not found");
1728 return rte_flow_error_set(error, EINVAL,
1729 RTE_FLOW_ERROR_TYPE_ACTION,
1730 (void *)items->type,
1731 "ip header not found");
1732 if (ipv4 && !ipv4->next_proto_id)
1733 ipv4->next_proto_id = IPPROTO_GRE;
1734 else if (ipv6 && !ipv6->proto)
1735 ipv6->proto = IPPROTO_GRE;
1737 case RTE_FLOW_ITEM_TYPE_VOID:
1740 return rte_flow_error_set(error, EINVAL,
1741 RTE_FLOW_ERROR_TYPE_ACTION,
1742 (void *)items->type,
1743 "unsupported item type");
1753 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1755 struct rte_ether_hdr *eth = NULL;
1756 struct rte_vlan_hdr *vlan = NULL;
1757 struct rte_ipv6_hdr *ipv6 = NULL;
1758 struct rte_udp_hdr *udp = NULL;
1762 eth = (struct rte_ether_hdr *)data;
1763 next_hdr = (char *)(eth + 1);
1764 proto = RTE_BE16(eth->ether_type);
1767 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1768 vlan = (struct rte_vlan_hdr *)next_hdr;
1769 proto = RTE_BE16(vlan->eth_proto);
1770 next_hdr += sizeof(struct rte_vlan_hdr);
1773 /* HW calculates IPv4 csum. no need to proceed */
1774 if (proto == RTE_ETHER_TYPE_IPV4)
1777 /* non IPv4/IPv6 header. not supported */
1778 if (proto != RTE_ETHER_TYPE_IPV6) {
1779 return rte_flow_error_set(error, ENOTSUP,
1780 RTE_FLOW_ERROR_TYPE_ACTION,
1781 NULL, "Cannot offload non IPv4/IPv6");
1784 ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1786 /* ignore non UDP */
1787 if (ipv6->proto != IPPROTO_UDP)
1790 udp = (struct rte_udp_hdr *)(ipv6 + 1);
1791 udp->dgram_cksum = 0;
1797 * Convert L2 encap action to DV specification.
1800 * Pointer to rte_eth_dev structure.
1802 * Pointer to action structure.
1803 * @param[in, out] dev_flow
1804 * Pointer to the mlx5_flow.
1805 * @param[in] transfer
1806 * Mark if the flow is E-Switch flow.
1808 * Pointer to the error structure.
1811 * 0 on success, a negative errno value otherwise and rte_errno is set.
1814 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1815 const struct rte_flow_action *action,
1816 struct mlx5_flow *dev_flow,
1818 struct rte_flow_error *error)
1820 const struct rte_flow_item *encap_data;
1821 const struct rte_flow_action_raw_encap *raw_encap_data;
1822 struct mlx5_flow_dv_encap_decap_resource res = {
1824 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1825 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1826 MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1829 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1831 (const struct rte_flow_action_raw_encap *)action->conf;
1832 res.size = raw_encap_data->size;
1833 memcpy(res.buf, raw_encap_data->data, res.size);
1834 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1837 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1839 ((const struct rte_flow_action_vxlan_encap *)
1840 action->conf)->definition;
1843 ((const struct rte_flow_action_nvgre_encap *)
1844 action->conf)->definition;
1845 if (flow_dv_convert_encap_data(encap_data, res.buf,
1849 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1850 return rte_flow_error_set(error, EINVAL,
1851 RTE_FLOW_ERROR_TYPE_ACTION,
1852 NULL, "can't create L2 encap action");
1857 * Convert L2 decap action to DV specification.
1860 * Pointer to rte_eth_dev structure.
1861 * @param[in, out] dev_flow
1862 * Pointer to the mlx5_flow.
1863 * @param[in] transfer
1864 * Mark if the flow is E-Switch flow.
1866 * Pointer to the error structure.
1869 * 0 on success, a negative errno value otherwise and rte_errno is set.
1872 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1873 struct mlx5_flow *dev_flow,
1875 struct rte_flow_error *error)
1877 struct mlx5_flow_dv_encap_decap_resource res = {
1880 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1881 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1882 MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1885 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1886 return rte_flow_error_set(error, EINVAL,
1887 RTE_FLOW_ERROR_TYPE_ACTION,
1888 NULL, "can't create L2 decap action");
1893 * Convert raw decap/encap (L3 tunnel) action to DV specification.
1896 * Pointer to rte_eth_dev structure.
1898 * Pointer to action structure.
1899 * @param[in, out] dev_flow
1900 * Pointer to the mlx5_flow.
1902 * Pointer to the flow attributes.
1904 * Pointer to the error structure.
1907 * 0 on success, a negative errno value otherwise and rte_errno is set.
1910 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1911 const struct rte_flow_action *action,
1912 struct mlx5_flow *dev_flow,
1913 const struct rte_flow_attr *attr,
1914 struct rte_flow_error *error)
1916 const struct rte_flow_action_raw_encap *encap_data;
1917 struct mlx5_flow_dv_encap_decap_resource res;
1919 encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1920 res.size = encap_data->size;
1921 memcpy(res.buf, encap_data->data, res.size);
1922 res.reformat_type = attr->egress ?
1923 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1924 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1926 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1928 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1929 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1930 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1931 return rte_flow_error_set(error, EINVAL,
1932 RTE_FLOW_ERROR_TYPE_ACTION,
1933 NULL, "can't create encap action");
1938 * Create action push VLAN.
1941 * Pointer to rte_eth_dev structure.
1942 * @param[in] vlan_tag
1943 * the vlan tag to push to the Ethernet header.
1944 * @param[in, out] dev_flow
1945 * Pointer to the mlx5_flow.
1947 * Pointer to the flow attributes.
1949 * Pointer to the error structure.
1952 * 0 on success, a negative errno value otherwise and rte_errno is set.
1955 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
1956 const struct rte_flow_attr *attr,
1957 const struct rte_vlan_hdr *vlan,
1958 struct mlx5_flow *dev_flow,
1959 struct rte_flow_error *error)
1961 struct mlx5_flow_dv_push_vlan_action_resource res;
1964 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
1967 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1969 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1970 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1971 return flow_dv_push_vlan_action_resource_register
1972 (dev, &res, dev_flow, error);
1976 * Validate the modify-header actions.
1978 * @param[in] action_flags
1979 * Holds the actions detected until now.
1981 * Pointer to the modify action.
1983 * Pointer to error structure.
1986 * 0 on success, a negative errno value otherwise and rte_errno is set.
1989 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1990 const struct rte_flow_action *action,
1991 struct rte_flow_error *error)
1993 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1994 return rte_flow_error_set(error, EINVAL,
1995 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1996 NULL, "action configuration not set");
1997 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1998 return rte_flow_error_set(error, EINVAL,
1999 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2000 "can't have encap action before"
2006 * Validate the modify-header MAC address actions.
2008 * @param[in] action_flags
2009 * Holds the actions detected until now.
2011 * Pointer to the modify action.
2012 * @param[in] item_flags
2013 * Holds the items detected.
2015 * Pointer to error structure.
2018 * 0 on success, a negative errno value otherwise and rte_errno is set.
2021 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
2022 const struct rte_flow_action *action,
2023 const uint64_t item_flags,
2024 struct rte_flow_error *error)
2028 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2030 if (!(item_flags & MLX5_FLOW_LAYER_L2))
2031 return rte_flow_error_set(error, EINVAL,
2032 RTE_FLOW_ERROR_TYPE_ACTION,
2034 "no L2 item in pattern");
2040 * Validate the modify-header IPv4 address actions.
2042 * @param[in] action_flags
2043 * Holds the actions detected until now.
2045 * Pointer to the modify action.
2046 * @param[in] item_flags
2047 * Holds the items detected.
2049 * Pointer to error structure.
2052 * 0 on success, a negative errno value otherwise and rte_errno is set.
2055 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
2056 const struct rte_flow_action *action,
2057 const uint64_t item_flags,
2058 struct rte_flow_error *error)
2062 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2064 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
2065 return rte_flow_error_set(error, EINVAL,
2066 RTE_FLOW_ERROR_TYPE_ACTION,
2068 "no ipv4 item in pattern");
2074 * Validate the modify-header IPv6 address actions.
2076 * @param[in] action_flags
2077 * Holds the actions detected until now.
2079 * Pointer to the modify action.
2080 * @param[in] item_flags
2081 * Holds the items detected.
2083 * Pointer to error structure.
2086 * 0 on success, a negative errno value otherwise and rte_errno is set.
2089 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
2090 const struct rte_flow_action *action,
2091 const uint64_t item_flags,
2092 struct rte_flow_error *error)
2096 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2098 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
2099 return rte_flow_error_set(error, EINVAL,
2100 RTE_FLOW_ERROR_TYPE_ACTION,
2102 "no ipv6 item in pattern");
2108 * Validate the modify-header TP actions.
2110 * @param[in] action_flags
2111 * Holds the actions detected until now.
2113 * Pointer to the modify action.
2114 * @param[in] item_flags
2115 * Holds the items detected.
2117 * Pointer to error structure.
2120 * 0 on success, a negative errno value otherwise and rte_errno is set.
2123 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
2124 const struct rte_flow_action *action,
2125 const uint64_t item_flags,
2126 struct rte_flow_error *error)
2130 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2132 if (!(item_flags & MLX5_FLOW_LAYER_L4))
2133 return rte_flow_error_set(error, EINVAL,
2134 RTE_FLOW_ERROR_TYPE_ACTION,
2135 NULL, "no transport layer "
2142 * Validate the modify-header actions of increment/decrement
2143 * TCP Sequence-number.
2145 * @param[in] action_flags
2146 * Holds the actions detected until now.
2148 * Pointer to the modify action.
2149 * @param[in] item_flags
2150 * Holds the items detected.
2152 * Pointer to error structure.
2155 * 0 on success, a negative errno value otherwise and rte_errno is set.
2158 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
2159 const struct rte_flow_action *action,
2160 const uint64_t item_flags,
2161 struct rte_flow_error *error)
2165 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2167 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2168 return rte_flow_error_set(error, EINVAL,
2169 RTE_FLOW_ERROR_TYPE_ACTION,
2170 NULL, "no TCP item in"
2172 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
2173 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
2174 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
2175 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
2176 return rte_flow_error_set(error, EINVAL,
2177 RTE_FLOW_ERROR_TYPE_ACTION,
2179 "cannot decrease and increase"
2180 " TCP sequence number"
2181 " at the same time");
2187 * Validate the modify-header actions of increment/decrement
2188 * TCP Acknowledgment number.
2190 * @param[in] action_flags
2191 * Holds the actions detected until now.
2193 * Pointer to the modify action.
2194 * @param[in] item_flags
2195 * Holds the items detected.
2197 * Pointer to error structure.
2200 * 0 on success, a negative errno value otherwise and rte_errno is set.
2203 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
2204 const struct rte_flow_action *action,
2205 const uint64_t item_flags,
2206 struct rte_flow_error *error)
2210 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2212 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2213 return rte_flow_error_set(error, EINVAL,
2214 RTE_FLOW_ERROR_TYPE_ACTION,
2215 NULL, "no TCP item in"
2217 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
2218 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
2219 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
2220 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
2221 return rte_flow_error_set(error, EINVAL,
2222 RTE_FLOW_ERROR_TYPE_ACTION,
2224 "cannot decrease and increase"
2225 " TCP acknowledgment number"
2226 " at the same time");
2232 * Validate the modify-header TTL actions.
2234 * @param[in] action_flags
2235 * Holds the actions detected until now.
2237 * Pointer to the modify action.
2238 * @param[in] item_flags
2239 * Holds the items detected.
2241 * Pointer to error structure.
2244 * 0 on success, a negative errno value otherwise and rte_errno is set.
2247 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
2248 const struct rte_flow_action *action,
2249 const uint64_t item_flags,
2250 struct rte_flow_error *error)
2254 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2256 if (!(item_flags & MLX5_FLOW_LAYER_L3))
2257 return rte_flow_error_set(error, EINVAL,
2258 RTE_FLOW_ERROR_TYPE_ACTION,
2260 "no IP protocol in pattern");
2266 * Validate jump action.
2269 * Pointer to the jump action.
2270 * @param[in] action_flags
2271 * Holds the actions detected until now.
2273 * The group of the current flow.
2275 * Pointer to error structure.
2278 * 0 on success, a negative errno value otherwise and rte_errno is set.
2281 flow_dv_validate_action_jump(const struct rte_flow_action *action,
2282 uint64_t action_flags,
2284 struct rte_flow_error *error)
2286 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2287 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2288 return rte_flow_error_set(error, EINVAL,
2289 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2290 "can't have 2 fate actions in"
2293 return rte_flow_error_set(error, EINVAL,
2294 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2295 NULL, "action configuration not set");
2296 if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
2297 return rte_flow_error_set(error, EINVAL,
2298 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2299 "target group must be higher then"
2300 " the current flow group");
2305 * Validate the port_id action.
2308 * Pointer to rte_eth_dev structure.
2309 * @param[in] action_flags
2310 * Bit-fields that holds the actions detected until now.
2312 * Port_id RTE action structure.
2314 * Attributes of flow that includes this action.
2316 * Pointer to error structure.
2319 * 0 on success, a negative errno value otherwise and rte_errno is set.
2322 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2323 uint64_t action_flags,
2324 const struct rte_flow_action *action,
2325 const struct rte_flow_attr *attr,
2326 struct rte_flow_error *error)
2328 const struct rte_flow_action_port_id *port_id;
2330 uint16_t esw_domain_id;
2331 uint16_t act_port_domain_id;
2334 if (!attr->transfer)
2335 return rte_flow_error_set(error, ENOTSUP,
2336 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2338 "port id action is valid in transfer"
2340 if (!action || !action->conf)
2341 return rte_flow_error_set(error, ENOTSUP,
2342 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2344 "port id action parameters must be"
2346 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2347 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2348 return rte_flow_error_set(error, EINVAL,
2349 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2350 "can have only one fate actions in"
2352 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2353 &esw_domain_id, NULL);
2355 return rte_flow_error_set(error, -ret,
2356 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2358 "failed to obtain E-Switch info");
2359 port_id = action->conf;
2360 port = port_id->original ? dev->data->port_id : port_id->id;
2361 ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2363 return rte_flow_error_set
2365 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2366 "failed to obtain E-Switch port id for port");
2367 if (act_port_domain_id != esw_domain_id)
2368 return rte_flow_error_set
2370 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2371 "port does not belong to"
2372 " E-Switch being configured");
2377 * Find existing modify-header resource or create and register a new one.
2379 * @param dev[in, out]
2380 * Pointer to rte_eth_dev structure.
2381 * @param[in, out] resource
2382 * Pointer to modify-header resource.
2383 * @parm[in, out] dev_flow
2384 * Pointer to the dev_flow.
2386 * pointer to error structure.
2389 * 0 on success otherwise -errno and errno is set.
2392 flow_dv_modify_hdr_resource_register
2393 (struct rte_eth_dev *dev,
2394 struct mlx5_flow_dv_modify_hdr_resource *resource,
2395 struct mlx5_flow *dev_flow,
2396 struct rte_flow_error *error)
2398 struct mlx5_priv *priv = dev->data->dev_private;
2399 struct mlx5_ibv_shared *sh = priv->sh;
2400 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2401 struct mlx5dv_dr_domain *ns;
2403 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2404 ns = sh->fdb_domain;
2405 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2410 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2411 /* Lookup a matching resource from cache. */
2412 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2413 if (resource->ft_type == cache_resource->ft_type &&
2414 resource->actions_num == cache_resource->actions_num &&
2415 resource->flags == cache_resource->flags &&
2416 !memcmp((const void *)resource->actions,
2417 (const void *)cache_resource->actions,
2418 (resource->actions_num *
2419 sizeof(resource->actions[0])))) {
2420 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2421 (void *)cache_resource,
2422 rte_atomic32_read(&cache_resource->refcnt));
2423 rte_atomic32_inc(&cache_resource->refcnt);
2424 dev_flow->dv.modify_hdr = cache_resource;
2428 /* Register new modify-header resource. */
2429 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2430 if (!cache_resource)
2431 return rte_flow_error_set(error, ENOMEM,
2432 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2433 "cannot allocate resource memory");
2434 *cache_resource = *resource;
2435 cache_resource->verbs_action =
2436 mlx5_glue->dv_create_flow_action_modify_header
2437 (sh->ctx, cache_resource->ft_type,
2438 ns, cache_resource->flags,
2439 cache_resource->actions_num *
2440 sizeof(cache_resource->actions[0]),
2441 (uint64_t *)cache_resource->actions);
2442 if (!cache_resource->verbs_action) {
2443 rte_free(cache_resource);
2444 return rte_flow_error_set(error, ENOMEM,
2445 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2446 NULL, "cannot create action");
2448 rte_atomic32_init(&cache_resource->refcnt);
2449 rte_atomic32_inc(&cache_resource->refcnt);
2450 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2451 dev_flow->dv.modify_hdr = cache_resource;
2452 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2453 (void *)cache_resource,
2454 rte_atomic32_read(&cache_resource->refcnt));
2458 #define MLX5_CNT_CONTAINER_RESIZE 64
2461 * Get or create a flow counter.
2464 * Pointer to the Ethernet device structure.
2466 * Indicate if this counter is shared with other flows.
2468 * Counter identifier.
2471 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2473 static struct mlx5_flow_counter *
2474 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2477 struct mlx5_priv *priv = dev->data->dev_private;
2478 struct mlx5_flow_counter *cnt = NULL;
2479 struct mlx5_devx_obj *dcs = NULL;
2481 if (!priv->config.devx) {
2482 rte_errno = ENOTSUP;
2486 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2487 if (cnt->shared && cnt->id == id) {
2493 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2496 cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2498 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2502 struct mlx5_flow_counter tmpl = {
2508 tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2510 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2516 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2521 * Release a flow counter.
2524 * Pointer to the Ethernet device structure.
2525 * @param[in] counter
2526 * Pointer to the counter handler.
2529 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2530 struct mlx5_flow_counter *counter)
2532 struct mlx5_priv *priv = dev->data->dev_private;
2536 if (--counter->ref_cnt == 0) {
2537 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2538 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2544 * Query a devx flow counter.
2547 * Pointer to the Ethernet device structure.
2549 * Pointer to the flow counter.
2551 * The statistics value of packets.
2553 * The statistics value of bytes.
2556 * 0 on success, otherwise a negative errno value and rte_errno is set.
2559 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2560 struct mlx5_flow_counter *cnt, uint64_t *pkts,
2563 return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2568 * Get a pool by a counter.
2571 * Pointer to the counter.
2576 static struct mlx5_flow_counter_pool *
2577 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2580 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2581 return (struct mlx5_flow_counter_pool *)cnt - 1;
2587 * Get a pool by devx counter ID.
2590 * Pointer to the counter container.
2592 * The counter devx ID.
2595 * The counter pool pointer if exists, NULL otherwise,
2597 static struct mlx5_flow_counter_pool *
2598 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2600 struct mlx5_flow_counter_pool *pool;
2602 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2603 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2604 MLX5_COUNTERS_PER_POOL;
2606 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2613 * Allocate a new memory for the counter values wrapped by all the needed
2617 * Pointer to the Ethernet device structure.
2619 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2622 * The new memory management pointer on success, otherwise NULL and rte_errno
2625 static struct mlx5_counter_stats_mem_mng *
2626 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2628 struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2629 (dev->data->dev_private))->sh;
2630 struct mlx5_devx_mkey_attr mkey_attr;
2631 struct mlx5_counter_stats_mem_mng *mem_mng;
2632 volatile struct flow_counter_stats *raw_data;
2633 int size = (sizeof(struct flow_counter_stats) *
2634 MLX5_COUNTERS_PER_POOL +
2635 sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2636 sizeof(struct mlx5_counter_stats_mem_mng);
2637 uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2644 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2645 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2646 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2647 IBV_ACCESS_LOCAL_WRITE);
2648 if (!mem_mng->umem) {
2653 mkey_attr.addr = (uintptr_t)mem;
2654 mkey_attr.size = size;
2655 mkey_attr.umem_id = mem_mng->umem->umem_id;
2656 mkey_attr.pd = sh->pdn;
2657 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2659 mlx5_glue->devx_umem_dereg(mem_mng->umem);
2664 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
2665 raw_data = (volatile struct flow_counter_stats *)mem;
2666 for (i = 0; i < raws_n; ++i) {
2667 mem_mng->raws[i].mem_mng = mem_mng;
2668 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
2670 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
2675 * Resize a counter container.
2678 * Pointer to the Ethernet device structure.
2680 * Whether the pool is for counter that was allocated by batch command.
2683 * The new container pointer on success, otherwise NULL and rte_errno is set.
2685 static struct mlx5_pools_container *
2686 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
2688 struct mlx5_priv *priv = dev->data->dev_private;
2689 struct mlx5_pools_container *cont =
2690 MLX5_CNT_CONTAINER(priv->sh, batch, 0);
2691 struct mlx5_pools_container *new_cont =
2692 MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
2693 struct mlx5_counter_stats_mem_mng *mem_mng;
2694 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
2695 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
2698 if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
2699 /* The last resize still hasn't detected by the host thread. */
2703 new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
2704 if (!new_cont->pools) {
2709 memcpy(new_cont->pools, cont->pools, cont->n *
2710 sizeof(struct mlx5_flow_counter_pool *));
2711 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
2712 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
2714 rte_free(new_cont->pools);
2717 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
2718 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
2719 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
2721 new_cont->n = resize;
2722 rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
2723 TAILQ_INIT(&new_cont->pool_list);
2724 TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
2725 new_cont->init_mem_mng = mem_mng;
2727 /* Flip the master container. */
2728 priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
2733 * Query a devx flow counter.
2736 * Pointer to the Ethernet device structure.
2738 * Pointer to the flow counter.
2740 * The statistics value of packets.
2742 * The statistics value of bytes.
2745 * 0 on success, otherwise a negative errno value and rte_errno is set.
2748 _flow_dv_query_count(struct rte_eth_dev *dev,
2749 struct mlx5_flow_counter *cnt, uint64_t *pkts,
2752 struct mlx5_priv *priv = dev->data->dev_private;
2753 struct mlx5_flow_counter_pool *pool =
2754 flow_dv_counter_pool_get(cnt);
2755 int offset = cnt - &pool->counters_raw[0];
2757 if (priv->counter_fallback)
2758 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
2760 rte_spinlock_lock(&pool->sl);
2762 * The single counters allocation may allocate smaller ID than the
2763 * current allocated in parallel to the host reading.
2764 * In this case the new counter values must be reported as 0.
2766 if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
2770 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
2771 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
2773 rte_spinlock_unlock(&pool->sl);
2778 * Create and initialize a new counter pool.
2781 * Pointer to the Ethernet device structure.
2783 * The devX counter handle.
2785 * Whether the pool is for counter that was allocated by batch command.
2788 * A new pool pointer on success, NULL otherwise and rte_errno is set.
2790 static struct mlx5_flow_counter_pool *
2791 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
2794 struct mlx5_priv *priv = dev->data->dev_private;
2795 struct mlx5_flow_counter_pool *pool;
2796 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2798 int16_t n_valid = rte_atomic16_read(&cont->n_valid);
2801 if (cont->n == n_valid) {
2802 cont = flow_dv_container_resize(dev, batch);
2806 size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
2807 sizeof(struct mlx5_flow_counter);
2808 pool = rte_calloc(__func__, 1, size, 0);
2813 pool->min_dcs = dcs;
2814 pool->raw = cont->init_mem_mng->raws + n_valid %
2815 MLX5_CNT_CONTAINER_RESIZE;
2816 pool->raw_hw = NULL;
2817 rte_spinlock_init(&pool->sl);
2819 * The generation of the new allocated counters in this pool is 0, 2 in
2820 * the pool generation makes all the counters valid for allocation.
2822 rte_atomic64_set(&pool->query_gen, 0x2);
2823 TAILQ_INIT(&pool->counters);
2824 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2825 cont->pools[n_valid] = pool;
2826 /* Pool initialization must be updated before host thread access. */
2828 rte_atomic16_add(&cont->n_valid, 1);
2833 * Prepare a new counter and/or a new counter pool.
2836 * Pointer to the Ethernet device structure.
2837 * @param[out] cnt_free
2838 * Where to put the pointer of a new counter.
2840 * Whether the pool is for counter that was allocated by batch command.
2843 * The free counter pool pointer and @p cnt_free is set on success,
2844 * NULL otherwise and rte_errno is set.
2846 static struct mlx5_flow_counter_pool *
2847 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
2848 struct mlx5_flow_counter **cnt_free,
2851 struct mlx5_priv *priv = dev->data->dev_private;
2852 struct mlx5_flow_counter_pool *pool;
2853 struct mlx5_devx_obj *dcs = NULL;
2854 struct mlx5_flow_counter *cnt;
2858 /* bulk_bitmap must be 0 for single counter allocation. */
2859 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2862 pool = flow_dv_find_pool_by_id
2863 (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
2865 pool = flow_dv_pool_create(dev, dcs, batch);
2867 mlx5_devx_cmd_destroy(dcs);
2870 } else if (dcs->id < pool->min_dcs->id) {
2871 rte_atomic64_set(&pool->a64_dcs,
2872 (int64_t)(uintptr_t)dcs);
2874 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
2875 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2880 /* bulk_bitmap is in 128 counters units. */
2881 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
2882 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
2884 rte_errno = ENODATA;
2887 pool = flow_dv_pool_create(dev, dcs, batch);
2889 mlx5_devx_cmd_destroy(dcs);
2892 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2893 cnt = &pool->counters_raw[i];
2895 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2897 *cnt_free = &pool->counters_raw[0];
2902 * Search for existed shared counter.
2905 * Pointer to the relevant counter pool container.
2907 * The shared counter ID to search.
2910 * NULL if not existed, otherwise pointer to the shared counter.
2912 static struct mlx5_flow_counter *
2913 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
2916 static struct mlx5_flow_counter *cnt;
2917 struct mlx5_flow_counter_pool *pool;
2920 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2921 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2922 cnt = &pool->counters_raw[i];
2923 if (cnt->ref_cnt && cnt->shared && cnt->id == id)
2931 * Allocate a flow counter.
2934 * Pointer to the Ethernet device structure.
2936 * Indicate if this counter is shared with other flows.
2938 * Counter identifier.
2940 * Counter flow group.
2943 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2945 static struct mlx5_flow_counter *
2946 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
2949 struct mlx5_priv *priv = dev->data->dev_private;
2950 struct mlx5_flow_counter_pool *pool = NULL;
2951 struct mlx5_flow_counter *cnt_free = NULL;
2953 * Currently group 0 flow counter cannot be assigned to a flow if it is
2954 * not the first one in the batch counter allocation, so it is better
2955 * to allocate counters one by one for these flows in a separate
2957 * A counter can be shared between different groups so need to take
2958 * shared counters from the single container.
2960 uint32_t batch = (group && !shared) ? 1 : 0;
2961 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2964 if (priv->counter_fallback)
2965 return flow_dv_counter_alloc_fallback(dev, shared, id);
2966 if (!priv->config.devx) {
2967 rte_errno = ENOTSUP;
2971 cnt_free = flow_dv_counter_shared_search(cont, id);
2973 if (cnt_free->ref_cnt + 1 == 0) {
2977 cnt_free->ref_cnt++;
2981 /* Pools which has a free counters are in the start. */
2982 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2984 * The free counter reset values must be updated between the
2985 * counter release to the counter allocation, so, at least one
2986 * query must be done in this time. ensure it by saving the
2987 * query generation in the release time.
2988 * The free list is sorted according to the generation - so if
2989 * the first one is not updated, all the others are not
2992 cnt_free = TAILQ_FIRST(&pool->counters);
2993 if (cnt_free && cnt_free->query_gen + 1 <
2994 rte_atomic64_read(&pool->query_gen))
2999 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
3003 cnt_free->batch = batch;
3004 /* Create a DV counter action only in the first time usage. */
3005 if (!cnt_free->action) {
3007 struct mlx5_devx_obj *dcs;
3010 offset = cnt_free - &pool->counters_raw[0];
3011 dcs = pool->min_dcs;
3014 dcs = cnt_free->dcs;
3016 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
3018 if (!cnt_free->action) {
3023 /* Update the counter reset values. */
3024 if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
3027 cnt_free->shared = shared;
3028 cnt_free->ref_cnt = 1;
3030 if (!priv->sh->cmng.query_thread_on)
3031 /* Start the asynchronous batch query by the host thread. */
3032 mlx5_set_query_alarm(priv->sh);
3033 TAILQ_REMOVE(&pool->counters, cnt_free, next);
3034 if (TAILQ_EMPTY(&pool->counters)) {
3035 /* Move the pool to the end of the container pool list. */
3036 TAILQ_REMOVE(&cont->pool_list, pool, next);
3037 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
3043 * Release a flow counter.
3046 * Pointer to the Ethernet device structure.
3047 * @param[in] counter
3048 * Pointer to the counter handler.
3051 flow_dv_counter_release(struct rte_eth_dev *dev,
3052 struct mlx5_flow_counter *counter)
3054 struct mlx5_priv *priv = dev->data->dev_private;
3058 if (priv->counter_fallback) {
3059 flow_dv_counter_release_fallback(dev, counter);
3062 if (--counter->ref_cnt == 0) {
3063 struct mlx5_flow_counter_pool *pool =
3064 flow_dv_counter_pool_get(counter);
3066 /* Put the counter in the end - the last updated one. */
3067 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
3068 counter->query_gen = rte_atomic64_read(&pool->query_gen);
3073 * Verify the @p attributes will be correctly understood by the NIC and store
3074 * them in the @p flow if everything is correct.
3077 * Pointer to dev struct.
3078 * @param[in] attributes
3079 * Pointer to flow attributes
3081 * Pointer to error structure.
3084 * 0 on success, a negative errno value otherwise and rte_errno is set.
3087 flow_dv_validate_attributes(struct rte_eth_dev *dev,
3088 const struct rte_flow_attr *attributes,
3089 struct rte_flow_error *error)
3091 struct mlx5_priv *priv = dev->data->dev_private;
3092 uint32_t priority_max = priv->config.flow_prio - 1;
3094 #ifndef HAVE_MLX5DV_DR
3095 if (attributes->group)
3096 return rte_flow_error_set(error, ENOTSUP,
3097 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
3099 "groups is not supported");
3101 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
3102 attributes->priority >= priority_max)
3103 return rte_flow_error_set(error, ENOTSUP,
3104 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
3106 "priority out of range");
3107 if (attributes->transfer) {
3108 if (!priv->config.dv_esw_en)
3109 return rte_flow_error_set
3111 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3112 "E-Switch dr is not supported");
3113 if (!(priv->representor || priv->master))
3114 return rte_flow_error_set
3115 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3116 NULL, "E-Switch configuration can only be"
3117 " done by a master or a representor device");
3118 if (attributes->egress)
3119 return rte_flow_error_set
3121 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
3122 "egress is not supported");
3123 if (attributes->group >= MLX5_MAX_TABLES_FDB)
3124 return rte_flow_error_set
3126 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
3127 NULL, "group must be smaller than "
3128 RTE_STR(MLX5_MAX_TABLES_FDB));
3130 if (!(attributes->egress ^ attributes->ingress))
3131 return rte_flow_error_set(error, ENOTSUP,
3132 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
3133 "must specify exactly one of "
3134 "ingress or egress");
3139 * Internal validation function. For validating both actions and items.
3142 * Pointer to the rte_eth_dev structure.
3144 * Pointer to the flow attributes.
3146 * Pointer to the list of items.
3147 * @param[in] actions
3148 * Pointer to the list of actions.
3150 * Pointer to the error structure.
3153 * 0 on success, a negative errno value otherwise and rte_errno is set.
3156 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
3157 const struct rte_flow_item items[],
3158 const struct rte_flow_action actions[],
3159 struct rte_flow_error *error)
3162 uint64_t action_flags = 0;
3163 uint64_t item_flags = 0;
3164 uint64_t last_item = 0;
3165 uint8_t next_protocol = 0xff;
3167 const struct rte_flow_item *gre_item = NULL;
3168 struct rte_flow_item_tcp nic_tcp_mask = {
3171 .src_port = RTE_BE16(UINT16_MAX),
3172 .dst_port = RTE_BE16(UINT16_MAX),
3178 ret = flow_dv_validate_attributes(dev, attr, error);
3181 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3182 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3183 switch (items->type) {
3184 case RTE_FLOW_ITEM_TYPE_VOID:
3186 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3187 ret = flow_dv_validate_item_port_id
3188 (dev, items, attr, item_flags, error);
3191 last_item = MLX5_FLOW_ITEM_PORT_ID;
3193 case RTE_FLOW_ITEM_TYPE_ETH:
3194 ret = mlx5_flow_validate_item_eth(items, item_flags,
3198 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
3199 MLX5_FLOW_LAYER_OUTER_L2;
3201 case RTE_FLOW_ITEM_TYPE_VLAN:
3202 ret = mlx5_flow_validate_item_vlan(items, item_flags,
3206 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
3207 MLX5_FLOW_LAYER_OUTER_VLAN;
3209 case RTE_FLOW_ITEM_TYPE_IPV4:
3210 mlx5_flow_tunnel_ip_check(items, next_protocol,
3211 &item_flags, &tunnel);
3212 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
3216 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3217 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3218 if (items->mask != NULL &&
3219 ((const struct rte_flow_item_ipv4 *)
3220 items->mask)->hdr.next_proto_id) {
3222 ((const struct rte_flow_item_ipv4 *)
3223 (items->spec))->hdr.next_proto_id;
3225 ((const struct rte_flow_item_ipv4 *)
3226 (items->mask))->hdr.next_proto_id;
3228 /* Reset for inner layer. */
3229 next_protocol = 0xff;
3232 case RTE_FLOW_ITEM_TYPE_IPV6:
3233 mlx5_flow_tunnel_ip_check(items, next_protocol,
3234 &item_flags, &tunnel);
3235 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
3239 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3240 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3241 if (items->mask != NULL &&
3242 ((const struct rte_flow_item_ipv6 *)
3243 items->mask)->hdr.proto) {
3245 ((const struct rte_flow_item_ipv6 *)
3246 items->spec)->hdr.proto;
3248 ((const struct rte_flow_item_ipv6 *)
3249 items->mask)->hdr.proto;
3251 /* Reset for inner layer. */
3252 next_protocol = 0xff;
3255 case RTE_FLOW_ITEM_TYPE_TCP:
3256 ret = mlx5_flow_validate_item_tcp
3263 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3264 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3266 case RTE_FLOW_ITEM_TYPE_UDP:
3267 ret = mlx5_flow_validate_item_udp(items, item_flags,
3272 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3273 MLX5_FLOW_LAYER_OUTER_L4_UDP;
3275 case RTE_FLOW_ITEM_TYPE_GRE:
3276 ret = mlx5_flow_validate_item_gre(items, item_flags,
3277 next_protocol, error);
3281 last_item = MLX5_FLOW_LAYER_GRE;
3283 case RTE_FLOW_ITEM_TYPE_NVGRE:
3284 ret = mlx5_flow_validate_item_nvgre(items, item_flags,
3289 last_item = MLX5_FLOW_LAYER_NVGRE;
3291 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
3292 ret = mlx5_flow_validate_item_gre_key
3293 (items, item_flags, gre_item, error);
3296 last_item = MLX5_FLOW_LAYER_GRE_KEY;
3298 case RTE_FLOW_ITEM_TYPE_VXLAN:
3299 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
3303 last_item = MLX5_FLOW_LAYER_VXLAN;
3305 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3306 ret = mlx5_flow_validate_item_vxlan_gpe(items,
3311 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3313 case RTE_FLOW_ITEM_TYPE_MPLS:
3314 ret = mlx5_flow_validate_item_mpls(dev, items,
3319 last_item = MLX5_FLOW_LAYER_MPLS;
3321 case RTE_FLOW_ITEM_TYPE_META:
3322 ret = flow_dv_validate_item_meta(dev, items, attr,
3326 last_item = MLX5_FLOW_ITEM_METADATA;
3328 case RTE_FLOW_ITEM_TYPE_ICMP:
3329 ret = mlx5_flow_validate_item_icmp(items, item_flags,
3334 last_item = MLX5_FLOW_LAYER_ICMP;
3336 case RTE_FLOW_ITEM_TYPE_ICMP6:
3337 ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3342 last_item = MLX5_FLOW_LAYER_ICMP6;
3345 return rte_flow_error_set(error, ENOTSUP,
3346 RTE_FLOW_ERROR_TYPE_ITEM,
3347 NULL, "item not supported");
3349 item_flags |= last_item;
3351 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3352 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3353 return rte_flow_error_set(error, ENOTSUP,
3354 RTE_FLOW_ERROR_TYPE_ACTION,
3355 actions, "too many actions");
3356 switch (actions->type) {
3357 case RTE_FLOW_ACTION_TYPE_VOID:
3359 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3360 ret = flow_dv_validate_action_port_id(dev,
3367 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3370 case RTE_FLOW_ACTION_TYPE_FLAG:
3371 ret = mlx5_flow_validate_action_flag(action_flags,
3375 action_flags |= MLX5_FLOW_ACTION_FLAG;
3378 case RTE_FLOW_ACTION_TYPE_MARK:
3379 ret = mlx5_flow_validate_action_mark(actions,
3384 action_flags |= MLX5_FLOW_ACTION_MARK;
3387 case RTE_FLOW_ACTION_TYPE_DROP:
3388 ret = mlx5_flow_validate_action_drop(action_flags,
3392 action_flags |= MLX5_FLOW_ACTION_DROP;
3395 case RTE_FLOW_ACTION_TYPE_QUEUE:
3396 ret = mlx5_flow_validate_action_queue(actions,
3401 action_flags |= MLX5_FLOW_ACTION_QUEUE;
3404 case RTE_FLOW_ACTION_TYPE_RSS:
3405 ret = mlx5_flow_validate_action_rss(actions,
3411 action_flags |= MLX5_FLOW_ACTION_RSS;
3414 case RTE_FLOW_ACTION_TYPE_COUNT:
3415 ret = flow_dv_validate_action_count(dev, error);
3418 action_flags |= MLX5_FLOW_ACTION_COUNT;
3421 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3422 if (flow_dv_validate_action_pop_vlan(dev,
3428 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
3431 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3432 ret = flow_dv_validate_action_push_vlan(action_flags,
3437 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
3440 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3441 ret = flow_dv_validate_action_set_vlan_pcp
3442 (action_flags, actions, error);
3445 /* Count PCP with push_vlan command. */
3447 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3448 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3449 ret = flow_dv_validate_action_l2_encap(action_flags,
3454 action_flags |= actions->type ==
3455 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3456 MLX5_FLOW_ACTION_VXLAN_ENCAP :
3457 MLX5_FLOW_ACTION_NVGRE_ENCAP;
3460 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3461 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3462 ret = flow_dv_validate_action_l2_decap(action_flags,
3466 action_flags |= actions->type ==
3467 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3468 MLX5_FLOW_ACTION_VXLAN_DECAP :
3469 MLX5_FLOW_ACTION_NVGRE_DECAP;
3472 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3473 ret = flow_dv_validate_action_raw_encap(action_flags,
3478 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3481 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3482 ret = flow_dv_validate_action_raw_decap(action_flags,
3487 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3490 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3491 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3492 ret = flow_dv_validate_action_modify_mac(action_flags,
3498 /* Count all modify-header actions as one action. */
3499 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3501 action_flags |= actions->type ==
3502 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3503 MLX5_FLOW_ACTION_SET_MAC_SRC :
3504 MLX5_FLOW_ACTION_SET_MAC_DST;
3507 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3508 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3509 ret = flow_dv_validate_action_modify_ipv4(action_flags,
3515 /* Count all modify-header actions as one action. */
3516 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3518 action_flags |= actions->type ==
3519 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3520 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3521 MLX5_FLOW_ACTION_SET_IPV4_DST;
3523 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3524 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3525 ret = flow_dv_validate_action_modify_ipv6(action_flags,
3531 /* Count all modify-header actions as one action. */
3532 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3534 action_flags |= actions->type ==
3535 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3536 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3537 MLX5_FLOW_ACTION_SET_IPV6_DST;
3539 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3540 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3541 ret = flow_dv_validate_action_modify_tp(action_flags,
3547 /* Count all modify-header actions as one action. */
3548 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3550 action_flags |= actions->type ==
3551 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3552 MLX5_FLOW_ACTION_SET_TP_SRC :
3553 MLX5_FLOW_ACTION_SET_TP_DST;
3555 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3556 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3557 ret = flow_dv_validate_action_modify_ttl(action_flags,
3563 /* Count all modify-header actions as one action. */
3564 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3566 action_flags |= actions->type ==
3567 RTE_FLOW_ACTION_TYPE_SET_TTL ?
3568 MLX5_FLOW_ACTION_SET_TTL :
3569 MLX5_FLOW_ACTION_DEC_TTL;
3571 case RTE_FLOW_ACTION_TYPE_JUMP:
3572 ret = flow_dv_validate_action_jump(actions,
3574 attr->group, error);
3578 action_flags |= MLX5_FLOW_ACTION_JUMP;
3580 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3581 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3582 ret = flow_dv_validate_action_modify_tcp_seq
3589 /* Count all modify-header actions as one action. */
3590 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3592 action_flags |= actions->type ==
3593 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
3594 MLX5_FLOW_ACTION_INC_TCP_SEQ :
3595 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
3597 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3598 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3599 ret = flow_dv_validate_action_modify_tcp_ack
3606 /* Count all modify-header actions as one action. */
3607 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3609 action_flags |= actions->type ==
3610 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
3611 MLX5_FLOW_ACTION_INC_TCP_ACK :
3612 MLX5_FLOW_ACTION_DEC_TCP_ACK;
3615 return rte_flow_error_set(error, ENOTSUP,
3616 RTE_FLOW_ERROR_TYPE_ACTION,
3618 "action not supported");
3621 if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3622 (action_flags & MLX5_FLOW_VLAN_ACTIONS))
3623 return rte_flow_error_set(error, ENOTSUP,
3624 RTE_FLOW_ERROR_TYPE_ACTION,
3626 "can't have vxlan and vlan"
3627 " actions in the same rule");
3628 /* Eswitch has few restrictions on using items and actions */
3629 if (attr->transfer) {
3630 if (action_flags & MLX5_FLOW_ACTION_FLAG)
3631 return rte_flow_error_set(error, ENOTSUP,
3632 RTE_FLOW_ERROR_TYPE_ACTION,
3634 "unsupported action FLAG");
3635 if (action_flags & MLX5_FLOW_ACTION_MARK)
3636 return rte_flow_error_set(error, ENOTSUP,
3637 RTE_FLOW_ERROR_TYPE_ACTION,
3639 "unsupported action MARK");
3640 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
3641 return rte_flow_error_set(error, ENOTSUP,
3642 RTE_FLOW_ERROR_TYPE_ACTION,
3644 "unsupported action QUEUE");
3645 if (action_flags & MLX5_FLOW_ACTION_RSS)
3646 return rte_flow_error_set(error, ENOTSUP,
3647 RTE_FLOW_ERROR_TYPE_ACTION,
3649 "unsupported action RSS");
3650 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3651 return rte_flow_error_set(error, EINVAL,
3652 RTE_FLOW_ERROR_TYPE_ACTION,
3654 "no fate action is found");
3656 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
3657 return rte_flow_error_set(error, EINVAL,
3658 RTE_FLOW_ERROR_TYPE_ACTION,
3660 "no fate action is found");
3666 * Internal preparation function. Allocates the DV flow size,
3667 * this size is constant.
3670 * Pointer to the flow attributes.
3672 * Pointer to the list of items.
3673 * @param[in] actions
3674 * Pointer to the list of actions.
3676 * Pointer to the error structure.
3679 * Pointer to mlx5_flow object on success,
3680 * otherwise NULL and rte_errno is set.
3682 static struct mlx5_flow *
3683 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
3684 const struct rte_flow_item items[] __rte_unused,
3685 const struct rte_flow_action actions[] __rte_unused,
3686 struct rte_flow_error *error)
3688 uint32_t size = sizeof(struct mlx5_flow);
3689 struct mlx5_flow *flow;
3691 flow = rte_calloc(__func__, 1, size, 0);
3693 rte_flow_error_set(error, ENOMEM,
3694 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3695 "not enough memory to create flow");
3698 flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
3704 * Sanity check for match mask and value. Similar to check_valid_spec() in
3705 * kernel driver. If unmasked bit is present in value, it returns failure.
3708 * pointer to match mask buffer.
3709 * @param match_value
3710 * pointer to match value buffer.
3713 * 0 if valid, -EINVAL otherwise.
3716 flow_dv_check_valid_spec(void *match_mask, void *match_value)
3718 uint8_t *m = match_mask;
3719 uint8_t *v = match_value;
3722 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
3725 "match_value differs from match_criteria"
3726 " %p[%u] != %p[%u]",
3727 match_value, i, match_mask, i);
3736 * Add Ethernet item to matcher and to the value.
3738 * @param[in, out] matcher
3740 * @param[in, out] key
3741 * Flow matcher value.
3743 * Flow pattern to translate.
3745 * Item is inner pattern.
3748 flow_dv_translate_item_eth(void *matcher, void *key,
3749 const struct rte_flow_item *item, int inner)
3751 const struct rte_flow_item_eth *eth_m = item->mask;
3752 const struct rte_flow_item_eth *eth_v = item->spec;
3753 const struct rte_flow_item_eth nic_mask = {
3754 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3755 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3756 .type = RTE_BE16(0xffff),
3768 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3770 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3772 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3774 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3776 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
3777 ð_m->dst, sizeof(eth_m->dst));
3778 /* The value must be in the range of the mask. */
3779 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
3780 for (i = 0; i < sizeof(eth_m->dst); ++i)
3781 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
3782 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
3783 ð_m->src, sizeof(eth_m->src));
3784 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
3785 /* The value must be in the range of the mask. */
3786 for (i = 0; i < sizeof(eth_m->dst); ++i)
3787 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
3788 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3789 rte_be_to_cpu_16(eth_m->type));
3790 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
3791 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
3795 * Add VLAN item to matcher and to the value.
3797 * @param[in, out] dev_flow
3799 * @param[in, out] matcher
3801 * @param[in, out] key
3802 * Flow matcher value.
3804 * Flow pattern to translate.
3806 * Item is inner pattern.
3809 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
3810 void *matcher, void *key,
3811 const struct rte_flow_item *item,
3814 const struct rte_flow_item_vlan *vlan_m = item->mask;
3815 const struct rte_flow_item_vlan *vlan_v = item->spec;
3824 vlan_m = &rte_flow_item_vlan_mask;
3826 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3828 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3830 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3832 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3834 * This is workaround, masks are not supported,
3835 * and pre-validated.
3837 dev_flow->dv.vf_vlan.tag =
3838 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
3840 tci_m = rte_be_to_cpu_16(vlan_m->tci);
3841 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
3842 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
3843 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
3844 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
3845 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
3846 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
3847 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
3848 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
3849 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
3850 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3851 rte_be_to_cpu_16(vlan_m->inner_type));
3852 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
3853 rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
3857 * Add IPV4 item to matcher and to the value.
3859 * @param[in, out] matcher
3861 * @param[in, out] key
3862 * Flow matcher value.
3864 * Flow pattern to translate.
3866 * Item is inner pattern.
3868 * The group to insert the rule.
3871 flow_dv_translate_item_ipv4(void *matcher, void *key,
3872 const struct rte_flow_item *item,
3873 int inner, uint32_t group)
3875 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
3876 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
3877 const struct rte_flow_item_ipv4 nic_mask = {
3879 .src_addr = RTE_BE32(0xffffffff),
3880 .dst_addr = RTE_BE32(0xffffffff),
3881 .type_of_service = 0xff,
3882 .next_proto_id = 0xff,
3892 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3894 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3896 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3898 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3901 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3903 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
3904 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
3909 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3910 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3911 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3912 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3913 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
3914 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
3915 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3916 src_ipv4_src_ipv6.ipv4_layout.ipv4);
3917 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3918 src_ipv4_src_ipv6.ipv4_layout.ipv4);
3919 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
3920 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
3921 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
3922 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
3923 ipv4_m->hdr.type_of_service);
3924 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
3925 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
3926 ipv4_m->hdr.type_of_service >> 2);
3927 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
3928 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3929 ipv4_m->hdr.next_proto_id);
3930 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3931 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3935 * Add IPV6 item to matcher and to the value.
3937 * @param[in, out] matcher
3939 * @param[in, out] key
3940 * Flow matcher value.
3942 * Flow pattern to translate.
3944 * Item is inner pattern.
3946 * The group to insert the rule.
3949 flow_dv_translate_item_ipv6(void *matcher, void *key,
3950 const struct rte_flow_item *item,
3951 int inner, uint32_t group)
3953 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
3954 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
3955 const struct rte_flow_item_ipv6 nic_mask = {
3958 "\xff\xff\xff\xff\xff\xff\xff\xff"
3959 "\xff\xff\xff\xff\xff\xff\xff\xff",
3961 "\xff\xff\xff\xff\xff\xff\xff\xff"
3962 "\xff\xff\xff\xff\xff\xff\xff\xff",
3963 .vtc_flow = RTE_BE32(0xffffffff),
3970 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3971 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3980 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3982 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3984 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3986 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3989 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3991 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
3992 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
3997 size = sizeof(ipv6_m->hdr.dst_addr);
3998 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3999 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
4000 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4001 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
4002 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
4003 for (i = 0; i < size; ++i)
4004 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
4005 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4006 src_ipv4_src_ipv6.ipv6_layout.ipv6);
4007 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4008 src_ipv4_src_ipv6.ipv6_layout.ipv6);
4009 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
4010 for (i = 0; i < size; ++i)
4011 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
4013 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
4014 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
4015 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
4016 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
4017 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
4018 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
4021 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
4023 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
4026 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
4028 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
4032 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
4034 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4035 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
4039 * Add TCP item to matcher and to the value.
4041 * @param[in, out] matcher
4043 * @param[in, out] key
4044 * Flow matcher value.
4046 * Flow pattern to translate.
4048 * Item is inner pattern.
4051 flow_dv_translate_item_tcp(void *matcher, void *key,
4052 const struct rte_flow_item *item,
4055 const struct rte_flow_item_tcp *tcp_m = item->mask;
4056 const struct rte_flow_item_tcp *tcp_v = item->spec;
4061 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4063 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4065 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4067 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4069 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4070 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
4074 tcp_m = &rte_flow_item_tcp_mask;
4075 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
4076 rte_be_to_cpu_16(tcp_m->hdr.src_port));
4077 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
4078 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
4079 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
4080 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
4081 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
4082 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
4083 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
4084 tcp_m->hdr.tcp_flags);
4085 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
4086 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
4090 * Add UDP item to matcher and to the value.
4092 * @param[in, out] matcher
4094 * @param[in, out] key
4095 * Flow matcher value.
4097 * Flow pattern to translate.
4099 * Item is inner pattern.
4102 flow_dv_translate_item_udp(void *matcher, void *key,
4103 const struct rte_flow_item *item,
4106 const struct rte_flow_item_udp *udp_m = item->mask;
4107 const struct rte_flow_item_udp *udp_v = item->spec;
4112 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4114 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4116 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4118 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4120 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4121 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
4125 udp_m = &rte_flow_item_udp_mask;
4126 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
4127 rte_be_to_cpu_16(udp_m->hdr.src_port));
4128 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
4129 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
4130 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
4131 rte_be_to_cpu_16(udp_m->hdr.dst_port));
4132 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4133 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
4137 * Add GRE optional Key item to matcher and to the value.
4139 * @param[in, out] matcher
4141 * @param[in, out] key
4142 * Flow matcher value.
4144 * Flow pattern to translate.
4146 * Item is inner pattern.
4149 flow_dv_translate_item_gre_key(void *matcher, void *key,
4150 const struct rte_flow_item *item)
4152 const rte_be32_t *key_m = item->mask;
4153 const rte_be32_t *key_v = item->spec;
4154 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4155 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4156 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
4161 key_m = &gre_key_default_mask;
4162 /* GRE K bit must be on and should already be validated */
4163 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
4164 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
4165 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
4166 rte_be_to_cpu_32(*key_m) >> 8);
4167 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
4168 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
4169 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
4170 rte_be_to_cpu_32(*key_m) & 0xFF);
4171 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
4172 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
4176 * Add GRE item to matcher and to the value.
4178 * @param[in, out] matcher
4180 * @param[in, out] key
4181 * Flow matcher value.
4183 * Flow pattern to translate.
4185 * Item is inner pattern.
4188 flow_dv_translate_item_gre(void *matcher, void *key,
4189 const struct rte_flow_item *item,
4192 const struct rte_flow_item_gre *gre_m = item->mask;
4193 const struct rte_flow_item_gre *gre_v = item->spec;
4196 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4197 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4204 uint16_t s_present:1;
4205 uint16_t k_present:1;
4206 uint16_t rsvd_bit1:1;
4207 uint16_t c_present:1;
4211 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
4214 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4216 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4218 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4220 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4222 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4223 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
4227 gre_m = &rte_flow_item_gre_mask;
4228 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
4229 rte_be_to_cpu_16(gre_m->protocol));
4230 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4231 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
4232 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
4233 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
4234 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
4235 gre_crks_rsvd0_ver_m.c_present);
4236 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
4237 gre_crks_rsvd0_ver_v.c_present &
4238 gre_crks_rsvd0_ver_m.c_present);
4239 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
4240 gre_crks_rsvd0_ver_m.k_present);
4241 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
4242 gre_crks_rsvd0_ver_v.k_present &
4243 gre_crks_rsvd0_ver_m.k_present);
4244 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
4245 gre_crks_rsvd0_ver_m.s_present);
4246 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
4247 gre_crks_rsvd0_ver_v.s_present &
4248 gre_crks_rsvd0_ver_m.s_present);
4252 * Add NVGRE item to matcher and to the value.
4254 * @param[in, out] matcher
4256 * @param[in, out] key
4257 * Flow matcher value.
4259 * Flow pattern to translate.
4261 * Item is inner pattern.
4264 flow_dv_translate_item_nvgre(void *matcher, void *key,
4265 const struct rte_flow_item *item,
4268 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
4269 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
4270 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4271 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4272 const char *tni_flow_id_m = (const char *)nvgre_m->tni;
4273 const char *tni_flow_id_v = (const char *)nvgre_v->tni;
4279 /* For NVGRE, GRE header fields must be set with defined values. */
4280 const struct rte_flow_item_gre gre_spec = {
4281 .c_rsvd0_ver = RTE_BE16(0x2000),
4282 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
4284 const struct rte_flow_item_gre gre_mask = {
4285 .c_rsvd0_ver = RTE_BE16(0xB000),
4286 .protocol = RTE_BE16(UINT16_MAX),
4288 const struct rte_flow_item gre_item = {
4293 flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
4297 nvgre_m = &rte_flow_item_nvgre_mask;
4298 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
4299 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
4300 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
4301 memcpy(gre_key_m, tni_flow_id_m, size);
4302 for (i = 0; i < size; ++i)
4303 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
4307 * Add VXLAN item to matcher and to the value.
4309 * @param[in, out] matcher
4311 * @param[in, out] key
4312 * Flow matcher value.
4314 * Flow pattern to translate.
4316 * Item is inner pattern.
4319 flow_dv_translate_item_vxlan(void *matcher, void *key,
4320 const struct rte_flow_item *item,
4323 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
4324 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
4327 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4328 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4336 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4338 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4340 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4342 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4344 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
4345 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
4346 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
4347 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
4348 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
4353 vxlan_m = &rte_flow_item_vxlan_mask;
4354 size = sizeof(vxlan_m->vni);
4355 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4356 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4357 memcpy(vni_m, vxlan_m->vni, size);
4358 for (i = 0; i < size; ++i)
4359 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4363 * Add MPLS item to matcher and to the value.
4365 * @param[in, out] matcher
4367 * @param[in, out] key
4368 * Flow matcher value.
4370 * Flow pattern to translate.
4371 * @param[in] prev_layer
4372 * The protocol layer indicated in previous item.
4374 * Item is inner pattern.
4377 flow_dv_translate_item_mpls(void *matcher, void *key,
4378 const struct rte_flow_item *item,
4379 uint64_t prev_layer,
4382 const uint32_t *in_mpls_m = item->mask;
4383 const uint32_t *in_mpls_v = item->spec;
4384 uint32_t *out_mpls_m = 0;
4385 uint32_t *out_mpls_v = 0;
4386 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4387 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4388 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4390 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4391 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4392 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4394 switch (prev_layer) {
4395 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4396 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4397 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4398 MLX5_UDP_PORT_MPLS);
4400 case MLX5_FLOW_LAYER_GRE:
4401 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4402 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4403 RTE_ETHER_TYPE_MPLS);
4406 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4407 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4414 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4415 switch (prev_layer) {
4416 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4418 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4419 outer_first_mpls_over_udp);
4421 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4422 outer_first_mpls_over_udp);
4424 case MLX5_FLOW_LAYER_GRE:
4426 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4427 outer_first_mpls_over_gre);
4429 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4430 outer_first_mpls_over_gre);
4433 /* Inner MPLS not over GRE is not supported. */
4436 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4440 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4446 if (out_mpls_m && out_mpls_v) {
4447 *out_mpls_m = *in_mpls_m;
4448 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4453 * Add META item to matcher
4455 * @param[in, out] matcher
4457 * @param[in, out] key
4458 * Flow matcher value.
4460 * Flow pattern to translate.
4462 * Item is inner pattern.
4465 flow_dv_translate_item_meta(void *matcher, void *key,
4466 const struct rte_flow_item *item)
4468 const struct rte_flow_item_meta *meta_m;
4469 const struct rte_flow_item_meta *meta_v;
4471 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4473 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4475 meta_m = (const void *)item->mask;
4477 meta_m = &rte_flow_item_meta_mask;
4478 meta_v = (const void *)item->spec;
4480 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
4481 rte_be_to_cpu_32(meta_m->data));
4482 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
4483 rte_be_to_cpu_32(meta_v->data & meta_m->data));
4488 * Add source vport match to the specified matcher.
4490 * @param[in, out] matcher
4492 * @param[in, out] key
4493 * Flow matcher value.
4495 * Source vport value to match
4500 flow_dv_translate_item_source_vport(void *matcher, void *key,
4501 int16_t port, uint16_t mask)
4503 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4504 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4506 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
4507 MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
4511 * Translate port-id item to eswitch match on port-id.
4514 * The devich to configure through.
4515 * @param[in, out] matcher
4517 * @param[in, out] key
4518 * Flow matcher value.
4520 * Flow pattern to translate.
4523 * 0 on success, a negative errno value otherwise.
4526 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
4527 void *key, const struct rte_flow_item *item)
4529 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
4530 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
4531 uint16_t mask, val, id;
4534 mask = pid_m ? pid_m->id : 0xffff;
4535 id = pid_v ? pid_v->id : dev->data->port_id;
4536 ret = mlx5_port_to_eswitch_info(id, NULL, &val);
4539 flow_dv_translate_item_source_vport(matcher, key, val, mask);
4544 * Add ICMP6 item to matcher and to the value.
4546 * @param[in, out] matcher
4548 * @param[in, out] key
4549 * Flow matcher value.
4551 * Flow pattern to translate.
4553 * Item is inner pattern.
4556 flow_dv_translate_item_icmp6(void *matcher, void *key,
4557 const struct rte_flow_item *item,
4560 const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
4561 const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
4564 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4566 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4568 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4570 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4572 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4574 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4576 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4577 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
4581 icmp6_m = &rte_flow_item_icmp6_mask;
4582 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
4583 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
4584 icmp6_v->type & icmp6_m->type);
4585 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
4586 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
4587 icmp6_v->code & icmp6_m->code);
4591 * Add ICMP item to matcher and to the value.
4593 * @param[in, out] matcher
4595 * @param[in, out] key
4596 * Flow matcher value.
4598 * Flow pattern to translate.
4600 * Item is inner pattern.
4603 flow_dv_translate_item_icmp(void *matcher, void *key,
4604 const struct rte_flow_item *item,
4607 const struct rte_flow_item_icmp *icmp_m = item->mask;
4608 const struct rte_flow_item_icmp *icmp_v = item->spec;
4611 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4613 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4615 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4617 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4619 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4621 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4623 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4624 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
4628 icmp_m = &rte_flow_item_icmp_mask;
4629 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
4630 icmp_m->hdr.icmp_type);
4631 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
4632 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
4633 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
4634 icmp_m->hdr.icmp_code);
4635 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
4636 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
4639 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
4641 #define HEADER_IS_ZERO(match_criteria, headers) \
4642 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
4643 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
4646 * Calculate flow matcher enable bitmap.
4648 * @param match_criteria
4649 * Pointer to flow matcher criteria.
4652 * Bitmap of enabled fields.
4655 flow_dv_matcher_enable(uint32_t *match_criteria)
4657 uint8_t match_criteria_enable;
4659 match_criteria_enable =
4660 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
4661 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
4662 match_criteria_enable |=
4663 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
4664 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
4665 match_criteria_enable |=
4666 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
4667 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
4668 match_criteria_enable |=
4669 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
4670 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
4671 match_criteria_enable |=
4672 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
4673 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
4674 return match_criteria_enable;
4681 * @param dev[in, out]
4682 * Pointer to rte_eth_dev structure.
4683 * @param[in] table_id
4686 * Direction of the table.
4687 * @param[in] transfer
4688 * E-Switch or NIC flow.
4690 * pointer to error structure.
4693 * Returns tables resource based on the index, NULL in case of failed.
4695 static struct mlx5_flow_tbl_resource *
4696 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
4697 uint32_t table_id, uint8_t egress,
4699 struct rte_flow_error *error)
4701 struct mlx5_priv *priv = dev->data->dev_private;
4702 struct mlx5_ibv_shared *sh = priv->sh;
4703 struct mlx5_flow_tbl_resource *tbl;
4705 #ifdef HAVE_MLX5DV_DR
4707 tbl = &sh->fdb_tbl[table_id];
4709 tbl->obj = mlx5_glue->dr_create_flow_tbl
4710 (sh->fdb_domain, table_id);
4711 } else if (egress) {
4712 tbl = &sh->tx_tbl[table_id];
4714 tbl->obj = mlx5_glue->dr_create_flow_tbl
4715 (sh->tx_domain, table_id);
4717 tbl = &sh->rx_tbl[table_id];
4719 tbl->obj = mlx5_glue->dr_create_flow_tbl
4720 (sh->rx_domain, table_id);
4723 rte_flow_error_set(error, ENOMEM,
4724 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4725 NULL, "cannot create table");
4728 rte_atomic32_inc(&tbl->refcnt);
4734 return &sh->fdb_tbl[table_id];
4736 return &sh->tx_tbl[table_id];
4738 return &sh->rx_tbl[table_id];
4743 * Release a flow table.
4746 * Table resource to be released.
4749 * Returns 0 if table was released, else return 1;
4752 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
4756 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
4757 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
4765 * Register the flow matcher.
4767 * @param dev[in, out]
4768 * Pointer to rte_eth_dev structure.
4769 * @param[in, out] matcher
4770 * Pointer to flow matcher.
4771 * @parm[in, out] dev_flow
4772 * Pointer to the dev_flow.
4774 * pointer to error structure.
4777 * 0 on success otherwise -errno and errno is set.
4780 flow_dv_matcher_register(struct rte_eth_dev *dev,
4781 struct mlx5_flow_dv_matcher *matcher,
4782 struct mlx5_flow *dev_flow,
4783 struct rte_flow_error *error)
4785 struct mlx5_priv *priv = dev->data->dev_private;
4786 struct mlx5_ibv_shared *sh = priv->sh;
4787 struct mlx5_flow_dv_matcher *cache_matcher;
4788 struct mlx5dv_flow_matcher_attr dv_attr = {
4789 .type = IBV_FLOW_ATTR_NORMAL,
4790 .match_mask = (void *)&matcher->mask,
4792 struct mlx5_flow_tbl_resource *tbl = NULL;
4794 /* Lookup from cache. */
4795 LIST_FOREACH(cache_matcher, &sh->matchers, next) {
4796 if (matcher->crc == cache_matcher->crc &&
4797 matcher->priority == cache_matcher->priority &&
4798 matcher->egress == cache_matcher->egress &&
4799 matcher->group == cache_matcher->group &&
4800 matcher->transfer == cache_matcher->transfer &&
4801 !memcmp((const void *)matcher->mask.buf,
4802 (const void *)cache_matcher->mask.buf,
4803 cache_matcher->mask.size)) {
4805 "priority %hd use %s matcher %p: refcnt %d++",
4806 cache_matcher->priority,
4807 cache_matcher->egress ? "tx" : "rx",
4808 (void *)cache_matcher,
4809 rte_atomic32_read(&cache_matcher->refcnt));
4810 rte_atomic32_inc(&cache_matcher->refcnt);
4811 dev_flow->dv.matcher = cache_matcher;
4815 /* Register new matcher. */
4816 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
4818 return rte_flow_error_set(error, ENOMEM,
4819 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4820 "cannot allocate matcher memory");
4821 tbl = flow_dv_tbl_resource_get(dev, matcher->group,
4822 matcher->egress, matcher->transfer,
4825 rte_free(cache_matcher);
4826 return rte_flow_error_set(error, ENOMEM,
4827 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4828 NULL, "cannot create table");
4830 *cache_matcher = *matcher;
4831 dv_attr.match_criteria_enable =
4832 flow_dv_matcher_enable(cache_matcher->mask.buf);
4833 dv_attr.priority = matcher->priority;
4834 if (matcher->egress)
4835 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
4836 cache_matcher->matcher_object =
4837 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
4838 if (!cache_matcher->matcher_object) {
4839 rte_free(cache_matcher);
4840 #ifdef HAVE_MLX5DV_DR
4841 flow_dv_tbl_resource_release(tbl);
4843 return rte_flow_error_set(error, ENOMEM,
4844 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4845 NULL, "cannot create matcher");
4847 rte_atomic32_inc(&cache_matcher->refcnt);
4848 LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
4849 dev_flow->dv.matcher = cache_matcher;
4850 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
4851 cache_matcher->priority,
4852 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
4853 rte_atomic32_read(&cache_matcher->refcnt));
4854 rte_atomic32_inc(&tbl->refcnt);
4859 * Find existing tag resource or create and register a new one.
4861 * @param dev[in, out]
4862 * Pointer to rte_eth_dev structure.
4863 * @param[in, out] resource
4864 * Pointer to tag resource.
4865 * @parm[in, out] dev_flow
4866 * Pointer to the dev_flow.
4868 * pointer to error structure.
4871 * 0 on success otherwise -errno and errno is set.
4874 flow_dv_tag_resource_register
4875 (struct rte_eth_dev *dev,
4876 struct mlx5_flow_dv_tag_resource *resource,
4877 struct mlx5_flow *dev_flow,
4878 struct rte_flow_error *error)
4880 struct mlx5_priv *priv = dev->data->dev_private;
4881 struct mlx5_ibv_shared *sh = priv->sh;
4882 struct mlx5_flow_dv_tag_resource *cache_resource;
4884 /* Lookup a matching resource from cache. */
4885 LIST_FOREACH(cache_resource, &sh->tags, next) {
4886 if (resource->tag == cache_resource->tag) {
4887 DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
4888 (void *)cache_resource,
4889 rte_atomic32_read(&cache_resource->refcnt));
4890 rte_atomic32_inc(&cache_resource->refcnt);
4891 dev_flow->flow->tag_resource = cache_resource;
4895 /* Register new resource. */
4896 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
4897 if (!cache_resource)
4898 return rte_flow_error_set(error, ENOMEM,
4899 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4900 "cannot allocate resource memory");
4901 *cache_resource = *resource;
4902 cache_resource->action = mlx5_glue->dv_create_flow_action_tag
4904 if (!cache_resource->action) {
4905 rte_free(cache_resource);
4906 return rte_flow_error_set(error, ENOMEM,
4907 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4908 NULL, "cannot create action");
4910 rte_atomic32_init(&cache_resource->refcnt);
4911 rte_atomic32_inc(&cache_resource->refcnt);
4912 LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
4913 dev_flow->flow->tag_resource = cache_resource;
4914 DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
4915 (void *)cache_resource,
4916 rte_atomic32_read(&cache_resource->refcnt));
4924 * Pointer to Ethernet device.
4926 * Pointer to mlx5_flow.
4929 * 1 while a reference on it exists, 0 when freed.
4932 flow_dv_tag_release(struct rte_eth_dev *dev,
4933 struct mlx5_flow_dv_tag_resource *tag)
4936 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
4937 dev->data->port_id, (void *)tag,
4938 rte_atomic32_read(&tag->refcnt));
4939 if (rte_atomic32_dec_and_test(&tag->refcnt)) {
4940 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
4941 LIST_REMOVE(tag, next);
4942 DRV_LOG(DEBUG, "port %u tag %p: removed",
4943 dev->data->port_id, (void *)tag);
4951 * Translate port ID action to vport.
4954 * Pointer to rte_eth_dev structure.
4956 * Pointer to the port ID action.
4957 * @param[out] dst_port_id
4958 * The target port ID.
4960 * Pointer to the error structure.
4963 * 0 on success, a negative errno value otherwise and rte_errno is set.
4966 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
4967 const struct rte_flow_action *action,
4968 uint32_t *dst_port_id,
4969 struct rte_flow_error *error)
4974 const struct rte_flow_action_port_id *conf =
4975 (const struct rte_flow_action_port_id *)action->conf;
4977 port = conf->original ? dev->data->port_id : conf->id;
4978 ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
4980 return rte_flow_error_set(error, -ret,
4981 RTE_FLOW_ERROR_TYPE_ACTION,
4983 "No eswitch info was found for port");
4984 *dst_port_id = port_id;
4989 * Fill the flow with DV spec.
4992 * Pointer to rte_eth_dev structure.
4993 * @param[in, out] dev_flow
4994 * Pointer to the sub flow.
4996 * Pointer to the flow attributes.
4998 * Pointer to the list of items.
4999 * @param[in] actions
5000 * Pointer to the list of actions.
5002 * Pointer to the error structure.
5005 * 0 on success, a negative errno value otherwise and rte_errno is set.
5008 flow_dv_translate(struct rte_eth_dev *dev,
5009 struct mlx5_flow *dev_flow,
5010 const struct rte_flow_attr *attr,
5011 const struct rte_flow_item items[],
5012 const struct rte_flow_action actions[],
5013 struct rte_flow_error *error)
5015 struct mlx5_priv *priv = dev->data->dev_private;
5016 struct rte_flow *flow = dev_flow->flow;
5017 uint64_t item_flags = 0;
5018 uint64_t last_item = 0;
5019 uint64_t action_flags = 0;
5020 uint64_t priority = attr->priority;
5021 struct mlx5_flow_dv_matcher matcher = {
5023 .size = sizeof(matcher.mask.buf),
5027 bool actions_end = false;
5028 struct mlx5_flow_dv_modify_hdr_resource res = {
5029 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
5030 MLX5DV_FLOW_TABLE_TYPE_NIC_RX
5032 union flow_dv_attr flow_attr = { .attr = 0 };
5033 struct mlx5_flow_dv_tag_resource tag_resource;
5034 uint32_t modify_action_position = UINT32_MAX;
5035 void *match_mask = matcher.mask.buf;
5036 void *match_value = dev_flow->dv.value.buf;
5037 uint8_t next_protocol = 0xff;
5038 struct rte_vlan_hdr vlan = { 0 };
5039 bool vlan_inherited = false;
5042 flow->group = attr->group;
5044 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
5045 if (priority == MLX5_FLOW_PRIO_RSVD)
5046 priority = priv->config.flow_prio - 1;
5047 for (; !actions_end ; actions++) {
5048 const struct rte_flow_action_queue *queue;
5049 const struct rte_flow_action_rss *rss;
5050 const struct rte_flow_action *action = actions;
5051 const struct rte_flow_action_count *count = action->conf;
5052 const uint8_t *rss_key;
5053 const struct rte_flow_action_jump *jump_data;
5054 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
5055 struct mlx5_flow_tbl_resource *tbl;
5056 uint32_t port_id = 0;
5057 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
5059 switch (actions->type) {
5060 case RTE_FLOW_ACTION_TYPE_VOID:
5062 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5063 if (flow_dv_translate_action_port_id(dev, action,
5066 port_id_resource.port_id = port_id;
5067 if (flow_dv_port_id_action_resource_register
5068 (dev, &port_id_resource, dev_flow, error))
5070 dev_flow->dv.actions[actions_n++] =
5071 dev_flow->dv.port_id_action->action;
5072 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5074 case RTE_FLOW_ACTION_TYPE_FLAG:
5076 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
5077 if (!flow->tag_resource)
5078 if (flow_dv_tag_resource_register
5079 (dev, &tag_resource, dev_flow, error))
5081 dev_flow->dv.actions[actions_n++] =
5082 flow->tag_resource->action;
5083 action_flags |= MLX5_FLOW_ACTION_FLAG;
5085 case RTE_FLOW_ACTION_TYPE_MARK:
5086 tag_resource.tag = mlx5_flow_mark_set
5087 (((const struct rte_flow_action_mark *)
5088 (actions->conf))->id);
5089 if (!flow->tag_resource)
5090 if (flow_dv_tag_resource_register
5091 (dev, &tag_resource, dev_flow, error))
5093 dev_flow->dv.actions[actions_n++] =
5094 flow->tag_resource->action;
5095 action_flags |= MLX5_FLOW_ACTION_MARK;
5097 case RTE_FLOW_ACTION_TYPE_DROP:
5098 action_flags |= MLX5_FLOW_ACTION_DROP;
5100 case RTE_FLOW_ACTION_TYPE_QUEUE:
5101 queue = actions->conf;
5102 flow->rss.queue_num = 1;
5103 (*flow->queue)[0] = queue->index;
5104 action_flags |= MLX5_FLOW_ACTION_QUEUE;
5106 case RTE_FLOW_ACTION_TYPE_RSS:
5107 rss = actions->conf;
5109 memcpy((*flow->queue), rss->queue,
5110 rss->queue_num * sizeof(uint16_t));
5111 flow->rss.queue_num = rss->queue_num;
5112 /* NULL RSS key indicates default RSS key. */
5113 rss_key = !rss->key ? rss_hash_default_key : rss->key;
5114 memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
5115 /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
5116 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
5117 flow->rss.level = rss->level;
5118 action_flags |= MLX5_FLOW_ACTION_RSS;
5120 case RTE_FLOW_ACTION_TYPE_COUNT:
5121 if (!priv->config.devx) {
5122 rte_errno = ENOTSUP;
5125 flow->counter = flow_dv_counter_alloc(dev,
5129 if (flow->counter == NULL)
5131 dev_flow->dv.actions[actions_n++] =
5132 flow->counter->action;
5133 action_flags |= MLX5_FLOW_ACTION_COUNT;
5136 if (rte_errno == ENOTSUP)
5137 return rte_flow_error_set
5139 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5141 "count action not supported");
5143 return rte_flow_error_set
5145 RTE_FLOW_ERROR_TYPE_ACTION,
5147 "cannot create counter"
5150 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5151 dev_flow->dv.actions[actions_n++] =
5152 priv->sh->pop_vlan_action;
5153 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5155 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5156 if (!vlan_inherited) {
5157 flow_dev_get_vlan_info_from_items(items, &vlan);
5158 vlan_inherited = true;
5160 vlan.eth_proto = rte_be_to_cpu_16
5161 ((((const struct rte_flow_action_of_push_vlan *)
5162 actions->conf)->ethertype));
5163 if (flow_dv_create_action_push_vlan
5164 (dev, attr, &vlan, dev_flow, error))
5166 dev_flow->dv.actions[actions_n++] =
5167 dev_flow->dv.push_vlan_res->action;
5168 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5170 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5171 if (!vlan_inherited) {
5172 flow_dev_get_vlan_info_from_items(items, &vlan);
5173 vlan_inherited = true;
5176 ((const struct rte_flow_action_of_set_vlan_pcp *)
5177 actions->conf)->vlan_pcp;
5178 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
5179 vlan.vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
5180 vlan.vlan_tci |= vlan_tci;
5181 /* Push VLAN command will use this value */
5183 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5184 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5185 if (flow_dv_create_action_l2_encap(dev, actions,
5190 dev_flow->dv.actions[actions_n++] =
5191 dev_flow->dv.encap_decap->verbs_action;
5192 action_flags |= actions->type ==
5193 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
5194 MLX5_FLOW_ACTION_VXLAN_ENCAP :
5195 MLX5_FLOW_ACTION_NVGRE_ENCAP;
5197 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5198 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5199 if (flow_dv_create_action_l2_decap(dev, dev_flow,
5203 dev_flow->dv.actions[actions_n++] =
5204 dev_flow->dv.encap_decap->verbs_action;
5205 action_flags |= actions->type ==
5206 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
5207 MLX5_FLOW_ACTION_VXLAN_DECAP :
5208 MLX5_FLOW_ACTION_NVGRE_DECAP;
5210 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5211 /* Handle encap with preceding decap. */
5212 if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
5213 if (flow_dv_create_action_raw_encap
5214 (dev, actions, dev_flow, attr, error))
5216 dev_flow->dv.actions[actions_n++] =
5217 dev_flow->dv.encap_decap->verbs_action;
5219 /* Handle encap without preceding decap. */
5220 if (flow_dv_create_action_l2_encap
5221 (dev, actions, dev_flow, attr->transfer,
5224 dev_flow->dv.actions[actions_n++] =
5225 dev_flow->dv.encap_decap->verbs_action;
5227 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
5229 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5230 /* Check if this decap is followed by encap. */
5231 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
5232 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
5235 /* Handle decap only if it isn't followed by encap. */
5236 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5237 if (flow_dv_create_action_l2_decap
5238 (dev, dev_flow, attr->transfer, error))
5240 dev_flow->dv.actions[actions_n++] =
5241 dev_flow->dv.encap_decap->verbs_action;
5243 /* If decap is followed by encap, handle it at encap. */
5244 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
5246 case RTE_FLOW_ACTION_TYPE_JUMP:
5247 jump_data = action->conf;
5248 tbl = flow_dv_tbl_resource_get(dev, jump_data->group,
5250 attr->transfer, error);
5252 return rte_flow_error_set
5254 RTE_FLOW_ERROR_TYPE_ACTION,
5256 "cannot create jump action.");
5257 jump_tbl_resource.tbl = tbl;
5258 if (flow_dv_jump_tbl_resource_register
5259 (dev, &jump_tbl_resource, dev_flow, error)) {
5260 flow_dv_tbl_resource_release(tbl);
5261 return rte_flow_error_set
5263 RTE_FLOW_ERROR_TYPE_ACTION,
5265 "cannot create jump action.");
5267 dev_flow->dv.actions[actions_n++] =
5268 dev_flow->dv.jump->action;
5269 action_flags |= MLX5_FLOW_ACTION_JUMP;
5271 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5272 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5273 if (flow_dv_convert_action_modify_mac(&res, actions,
5276 action_flags |= actions->type ==
5277 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5278 MLX5_FLOW_ACTION_SET_MAC_SRC :
5279 MLX5_FLOW_ACTION_SET_MAC_DST;
5281 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5282 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5283 if (flow_dv_convert_action_modify_ipv4(&res, actions,
5286 action_flags |= actions->type ==
5287 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5288 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5289 MLX5_FLOW_ACTION_SET_IPV4_DST;
5291 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5292 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5293 if (flow_dv_convert_action_modify_ipv6(&res, actions,
5296 action_flags |= actions->type ==
5297 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5298 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5299 MLX5_FLOW_ACTION_SET_IPV6_DST;
5301 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5302 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5303 if (flow_dv_convert_action_modify_tp(&res, actions,
5307 action_flags |= actions->type ==
5308 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5309 MLX5_FLOW_ACTION_SET_TP_SRC :
5310 MLX5_FLOW_ACTION_SET_TP_DST;
5312 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5313 if (flow_dv_convert_action_modify_dec_ttl(&res, items,
5317 action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
5319 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5320 if (flow_dv_convert_action_modify_ttl(&res, actions,
5324 action_flags |= MLX5_FLOW_ACTION_SET_TTL;
5326 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5327 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5328 if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
5331 action_flags |= actions->type ==
5332 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5333 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5334 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5337 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5338 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5339 if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
5342 action_flags |= actions->type ==
5343 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5344 MLX5_FLOW_ACTION_INC_TCP_ACK :
5345 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5347 case RTE_FLOW_ACTION_TYPE_END:
5349 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
5350 /* create modify action if needed. */
5351 if (flow_dv_modify_hdr_resource_register
5356 dev_flow->dv.actions[modify_action_position] =
5357 dev_flow->dv.modify_hdr->verbs_action;
5363 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
5364 modify_action_position == UINT32_MAX)
5365 modify_action_position = actions_n++;
5367 dev_flow->dv.actions_n = actions_n;
5368 flow->actions = action_flags;
5369 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5370 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5372 switch (items->type) {
5373 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5374 flow_dv_translate_item_port_id(dev, match_mask,
5375 match_value, items);
5376 last_item = MLX5_FLOW_ITEM_PORT_ID;
5378 case RTE_FLOW_ITEM_TYPE_ETH:
5379 flow_dv_translate_item_eth(match_mask, match_value,
5381 matcher.priority = MLX5_PRIORITY_MAP_L2;
5382 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5383 MLX5_FLOW_LAYER_OUTER_L2;
5385 case RTE_FLOW_ITEM_TYPE_VLAN:
5386 flow_dv_translate_item_vlan(dev_flow,
5387 match_mask, match_value,
5389 matcher.priority = MLX5_PRIORITY_MAP_L2;
5390 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
5391 MLX5_FLOW_LAYER_INNER_VLAN) :
5392 (MLX5_FLOW_LAYER_OUTER_L2 |
5393 MLX5_FLOW_LAYER_OUTER_VLAN);
5395 case RTE_FLOW_ITEM_TYPE_IPV4:
5396 mlx5_flow_tunnel_ip_check(items, next_protocol,
5397 &item_flags, &tunnel);
5398 flow_dv_translate_item_ipv4(match_mask, match_value,
5399 items, tunnel, attr->group);
5400 matcher.priority = MLX5_PRIORITY_MAP_L3;
5401 dev_flow->dv.hash_fields |=
5402 mlx5_flow_hashfields_adjust
5404 MLX5_IPV4_LAYER_TYPES,
5405 MLX5_IPV4_IBV_RX_HASH);
5406 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5407 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5408 if (items->mask != NULL &&
5409 ((const struct rte_flow_item_ipv4 *)
5410 items->mask)->hdr.next_proto_id) {
5412 ((const struct rte_flow_item_ipv4 *)
5413 (items->spec))->hdr.next_proto_id;
5415 ((const struct rte_flow_item_ipv4 *)
5416 (items->mask))->hdr.next_proto_id;
5418 /* Reset for inner layer. */
5419 next_protocol = 0xff;
5422 case RTE_FLOW_ITEM_TYPE_IPV6:
5423 mlx5_flow_tunnel_ip_check(items, next_protocol,
5424 &item_flags, &tunnel);
5425 flow_dv_translate_item_ipv6(match_mask, match_value,
5426 items, tunnel, attr->group);
5427 matcher.priority = MLX5_PRIORITY_MAP_L3;
5428 dev_flow->dv.hash_fields |=
5429 mlx5_flow_hashfields_adjust
5431 MLX5_IPV6_LAYER_TYPES,
5432 MLX5_IPV6_IBV_RX_HASH);
5433 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5434 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5435 if (items->mask != NULL &&
5436 ((const struct rte_flow_item_ipv6 *)
5437 items->mask)->hdr.proto) {
5439 ((const struct rte_flow_item_ipv6 *)
5440 items->spec)->hdr.proto;
5442 ((const struct rte_flow_item_ipv6 *)
5443 items->mask)->hdr.proto;
5445 /* Reset for inner layer. */
5446 next_protocol = 0xff;
5449 case RTE_FLOW_ITEM_TYPE_TCP:
5450 flow_dv_translate_item_tcp(match_mask, match_value,
5452 matcher.priority = MLX5_PRIORITY_MAP_L4;
5453 dev_flow->dv.hash_fields |=
5454 mlx5_flow_hashfields_adjust
5455 (dev_flow, tunnel, ETH_RSS_TCP,
5456 IBV_RX_HASH_SRC_PORT_TCP |
5457 IBV_RX_HASH_DST_PORT_TCP);
5458 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5459 MLX5_FLOW_LAYER_OUTER_L4_TCP;
5461 case RTE_FLOW_ITEM_TYPE_UDP:
5462 flow_dv_translate_item_udp(match_mask, match_value,
5464 matcher.priority = MLX5_PRIORITY_MAP_L4;
5465 dev_flow->dv.hash_fields |=
5466 mlx5_flow_hashfields_adjust
5467 (dev_flow, tunnel, ETH_RSS_UDP,
5468 IBV_RX_HASH_SRC_PORT_UDP |
5469 IBV_RX_HASH_DST_PORT_UDP);
5470 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5471 MLX5_FLOW_LAYER_OUTER_L4_UDP;
5473 case RTE_FLOW_ITEM_TYPE_GRE:
5474 flow_dv_translate_item_gre(match_mask, match_value,
5476 last_item = MLX5_FLOW_LAYER_GRE;
5478 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5479 flow_dv_translate_item_gre_key(match_mask,
5480 match_value, items);
5481 last_item = MLX5_FLOW_LAYER_GRE_KEY;
5483 case RTE_FLOW_ITEM_TYPE_NVGRE:
5484 flow_dv_translate_item_nvgre(match_mask, match_value,
5486 last_item = MLX5_FLOW_LAYER_GRE;
5488 case RTE_FLOW_ITEM_TYPE_VXLAN:
5489 flow_dv_translate_item_vxlan(match_mask, match_value,
5491 last_item = MLX5_FLOW_LAYER_VXLAN;
5493 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5494 flow_dv_translate_item_vxlan(match_mask, match_value,
5496 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5498 case RTE_FLOW_ITEM_TYPE_MPLS:
5499 flow_dv_translate_item_mpls(match_mask, match_value,
5500 items, last_item, tunnel);
5501 last_item = MLX5_FLOW_LAYER_MPLS;
5503 case RTE_FLOW_ITEM_TYPE_META:
5504 flow_dv_translate_item_meta(match_mask, match_value,
5506 last_item = MLX5_FLOW_ITEM_METADATA;
5508 case RTE_FLOW_ITEM_TYPE_ICMP:
5509 flow_dv_translate_item_icmp(match_mask, match_value,
5511 last_item = MLX5_FLOW_LAYER_ICMP;
5513 case RTE_FLOW_ITEM_TYPE_ICMP6:
5514 flow_dv_translate_item_icmp6(match_mask, match_value,
5516 last_item = MLX5_FLOW_LAYER_ICMP6;
5521 item_flags |= last_item;
5524 * In case of ingress traffic when E-Switch mode is enabled,
5525 * we have two cases where we need to set the source port manually.
5526 * The first one, is in case of Nic steering rule, and the second is
5527 * E-Switch rule where no port_id item was found. In both cases
5528 * the source port is set according the current port in use.
5530 if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
5531 (priv->representor || priv->master)) {
5532 if (flow_dv_translate_item_port_id(dev, match_mask,
5536 assert(!flow_dv_check_valid_spec(matcher.mask.buf,
5537 dev_flow->dv.value.buf));
5538 dev_flow->layers = item_flags;
5539 /* Register matcher. */
5540 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
5542 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
5544 matcher.egress = attr->egress;
5545 matcher.group = attr->group;
5546 matcher.transfer = attr->transfer;
5547 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
5553 * Apply the flow to the NIC.
5556 * Pointer to the Ethernet device structure.
5557 * @param[in, out] flow
5558 * Pointer to flow structure.
5560 * Pointer to error structure.
5563 * 0 on success, a negative errno value otherwise and rte_errno is set.
5566 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5567 struct rte_flow_error *error)
5569 struct mlx5_flow_dv *dv;
5570 struct mlx5_flow *dev_flow;
5571 struct mlx5_priv *priv = dev->data->dev_private;
5575 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5578 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
5579 if (flow->transfer) {
5580 dv->actions[n++] = priv->sh->esw_drop_action;
5582 dv->hrxq = mlx5_hrxq_drop_new(dev);
5586 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5588 "cannot get drop hash queue");
5591 dv->actions[n++] = dv->hrxq->action;
5593 } else if (flow->actions &
5594 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
5595 struct mlx5_hrxq *hrxq;
5597 hrxq = mlx5_hrxq_get(dev, flow->key,
5598 MLX5_RSS_HASH_KEY_LEN,
5601 flow->rss.queue_num);
5603 hrxq = mlx5_hrxq_new
5604 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
5605 dv->hash_fields, (*flow->queue),
5606 flow->rss.queue_num,
5607 !!(dev_flow->layers &
5608 MLX5_FLOW_LAYER_TUNNEL));
5613 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5614 "cannot get hash queue");
5618 dv->actions[n++] = dv->hrxq->action;
5621 mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
5622 (void *)&dv->value, n,
5625 rte_flow_error_set(error, errno,
5626 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5628 "hardware refuses to create flow");
5631 if (priv->vmwa_context &&
5632 dev_flow->dv.vf_vlan.tag &&
5633 !dev_flow->dv.vf_vlan.created) {
5635 * The rule contains the VLAN pattern.
5636 * For VF we are going to create VLAN
5637 * interface to make hypervisor set correct
5638 * e-Switch vport context.
5640 mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
5645 err = rte_errno; /* Save rte_errno before cleanup. */
5646 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5647 struct mlx5_flow_dv *dv = &dev_flow->dv;
5649 if (flow->actions & MLX5_FLOW_ACTION_DROP)
5650 mlx5_hrxq_drop_release(dev);
5652 mlx5_hrxq_release(dev, dv->hrxq);
5655 if (dev_flow->dv.vf_vlan.tag &&
5656 dev_flow->dv.vf_vlan.created)
5657 mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5659 rte_errno = err; /* Restore rte_errno. */
5664 * Release the flow matcher.
5667 * Pointer to Ethernet device.
5669 * Pointer to mlx5_flow.
5672 * 1 while a reference on it exists, 0 when freed.
5675 flow_dv_matcher_release(struct rte_eth_dev *dev,
5676 struct mlx5_flow *flow)
5678 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
5679 struct mlx5_priv *priv = dev->data->dev_private;
5680 struct mlx5_ibv_shared *sh = priv->sh;
5681 struct mlx5_flow_tbl_resource *tbl;
5683 assert(matcher->matcher_object);
5684 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
5685 dev->data->port_id, (void *)matcher,
5686 rte_atomic32_read(&matcher->refcnt));
5687 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
5688 claim_zero(mlx5_glue->dv_destroy_flow_matcher
5689 (matcher->matcher_object));
5690 LIST_REMOVE(matcher, next);
5691 if (matcher->egress)
5692 tbl = &sh->tx_tbl[matcher->group];
5694 tbl = &sh->rx_tbl[matcher->group];
5695 flow_dv_tbl_resource_release(tbl);
5697 DRV_LOG(DEBUG, "port %u matcher %p: removed",
5698 dev->data->port_id, (void *)matcher);
5705 * Release an encap/decap resource.
5708 * Pointer to mlx5_flow.
5711 * 1 while a reference on it exists, 0 when freed.
5714 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
5716 struct mlx5_flow_dv_encap_decap_resource *cache_resource =
5717 flow->dv.encap_decap;
5719 assert(cache_resource->verbs_action);
5720 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
5721 (void *)cache_resource,
5722 rte_atomic32_read(&cache_resource->refcnt));
5723 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5724 claim_zero(mlx5_glue->destroy_flow_action
5725 (cache_resource->verbs_action));
5726 LIST_REMOVE(cache_resource, next);
5727 rte_free(cache_resource);
5728 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
5729 (void *)cache_resource);
5736 * Release an jump to table action resource.
5739 * Pointer to mlx5_flow.
5742 * 1 while a reference on it exists, 0 when freed.
5745 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
5747 struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
5750 assert(cache_resource->action);
5751 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
5752 (void *)cache_resource,
5753 rte_atomic32_read(&cache_resource->refcnt));
5754 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5755 claim_zero(mlx5_glue->destroy_flow_action
5756 (cache_resource->action));
5757 LIST_REMOVE(cache_resource, next);
5758 flow_dv_tbl_resource_release(cache_resource->tbl);
5759 rte_free(cache_resource);
5760 DRV_LOG(DEBUG, "jump table resource %p: removed",
5761 (void *)cache_resource);
5768 * Release a modify-header resource.
5771 * Pointer to mlx5_flow.
5774 * 1 while a reference on it exists, 0 when freed.
5777 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
5779 struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
5780 flow->dv.modify_hdr;
5782 assert(cache_resource->verbs_action);
5783 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
5784 (void *)cache_resource,
5785 rte_atomic32_read(&cache_resource->refcnt));
5786 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5787 claim_zero(mlx5_glue->destroy_flow_action
5788 (cache_resource->verbs_action));
5789 LIST_REMOVE(cache_resource, next);
5790 rte_free(cache_resource);
5791 DRV_LOG(DEBUG, "modify-header resource %p: removed",
5792 (void *)cache_resource);
5799 * Release port ID action resource.
5802 * Pointer to mlx5_flow.
5805 * 1 while a reference on it exists, 0 when freed.
5808 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
5810 struct mlx5_flow_dv_port_id_action_resource *cache_resource =
5811 flow->dv.port_id_action;
5813 assert(cache_resource->action);
5814 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
5815 (void *)cache_resource,
5816 rte_atomic32_read(&cache_resource->refcnt));
5817 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5818 claim_zero(mlx5_glue->destroy_flow_action
5819 (cache_resource->action));
5820 LIST_REMOVE(cache_resource, next);
5821 rte_free(cache_resource);
5822 DRV_LOG(DEBUG, "port id action resource %p: removed",
5823 (void *)cache_resource);
5830 * Release push vlan action resource.
5833 * Pointer to mlx5_flow.
5836 * 1 while a reference on it exists, 0 when freed.
5839 flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow)
5841 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource =
5842 flow->dv.push_vlan_res;
5844 assert(cache_resource->action);
5845 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
5846 (void *)cache_resource,
5847 rte_atomic32_read(&cache_resource->refcnt));
5848 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5849 claim_zero(mlx5_glue->destroy_flow_action
5850 (cache_resource->action));
5851 LIST_REMOVE(cache_resource, next);
5852 rte_free(cache_resource);
5853 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
5854 (void *)cache_resource);
5861 * Remove the flow from the NIC but keeps it in memory.
5864 * Pointer to Ethernet device.
5865 * @param[in, out] flow
5866 * Pointer to flow structure.
5869 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5871 struct mlx5_flow_dv *dv;
5872 struct mlx5_flow *dev_flow;
5876 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5879 claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
5883 if (flow->actions & MLX5_FLOW_ACTION_DROP)
5884 mlx5_hrxq_drop_release(dev);
5886 mlx5_hrxq_release(dev, dv->hrxq);
5889 if (dev_flow->dv.vf_vlan.tag &&
5890 dev_flow->dv.vf_vlan.created)
5891 mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5896 * Remove the flow from the NIC and the memory.
5899 * Pointer to the Ethernet device structure.
5900 * @param[in, out] flow
5901 * Pointer to flow structure.
5904 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5906 struct mlx5_flow *dev_flow;
5910 flow_dv_remove(dev, flow);
5911 if (flow->counter) {
5912 flow_dv_counter_release(dev, flow->counter);
5913 flow->counter = NULL;
5915 if (flow->tag_resource) {
5916 flow_dv_tag_release(dev, flow->tag_resource);
5917 flow->tag_resource = NULL;
5919 while (!LIST_EMPTY(&flow->dev_flows)) {
5920 dev_flow = LIST_FIRST(&flow->dev_flows);
5921 LIST_REMOVE(dev_flow, next);
5922 if (dev_flow->dv.matcher)
5923 flow_dv_matcher_release(dev, dev_flow);
5924 if (dev_flow->dv.encap_decap)
5925 flow_dv_encap_decap_resource_release(dev_flow);
5926 if (dev_flow->dv.modify_hdr)
5927 flow_dv_modify_hdr_resource_release(dev_flow);
5928 if (dev_flow->dv.jump)
5929 flow_dv_jump_tbl_resource_release(dev_flow);
5930 if (dev_flow->dv.port_id_action)
5931 flow_dv_port_id_action_resource_release(dev_flow);
5932 if (dev_flow->dv.push_vlan_res)
5933 flow_dv_push_vlan_action_resource_release(dev_flow);
5939 * Query a dv flow rule for its statistics via devx.
5942 * Pointer to Ethernet device.
5944 * Pointer to the sub flow.
5946 * data retrieved by the query.
5948 * Perform verbose error reporting if not NULL.
5951 * 0 on success, a negative errno value otherwise and rte_errno is set.
5954 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
5955 void *data, struct rte_flow_error *error)
5957 struct mlx5_priv *priv = dev->data->dev_private;
5958 struct rte_flow_query_count *qc = data;
5960 if (!priv->config.devx)
5961 return rte_flow_error_set(error, ENOTSUP,
5962 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5964 "counters are not supported");
5965 if (flow->counter) {
5966 uint64_t pkts, bytes;
5967 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
5971 return rte_flow_error_set(error, -err,
5972 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5973 NULL, "cannot read counters");
5976 qc->hits = pkts - flow->counter->hits;
5977 qc->bytes = bytes - flow->counter->bytes;
5979 flow->counter->hits = pkts;
5980 flow->counter->bytes = bytes;
5984 return rte_flow_error_set(error, EINVAL,
5985 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5987 "counters are not available");
5993 * @see rte_flow_query()
5997 flow_dv_query(struct rte_eth_dev *dev,
5998 struct rte_flow *flow __rte_unused,
5999 const struct rte_flow_action *actions __rte_unused,
6000 void *data __rte_unused,
6001 struct rte_flow_error *error __rte_unused)
6005 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
6006 switch (actions->type) {
6007 case RTE_FLOW_ACTION_TYPE_VOID:
6009 case RTE_FLOW_ACTION_TYPE_COUNT:
6010 ret = flow_dv_query_count(dev, flow, data, error);
6013 return rte_flow_error_set(error, ENOTSUP,
6014 RTE_FLOW_ERROR_TYPE_ACTION,
6016 "action not supported");
6023 * Mutex-protected thunk to flow_dv_translate().
6026 flow_d_translate(struct rte_eth_dev *dev,
6027 struct mlx5_flow *dev_flow,
6028 const struct rte_flow_attr *attr,
6029 const struct rte_flow_item items[],
6030 const struct rte_flow_action actions[],
6031 struct rte_flow_error *error)
6035 flow_d_shared_lock(dev);
6036 ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
6037 flow_d_shared_unlock(dev);
6042 * Mutex-protected thunk to flow_dv_apply().
6045 flow_d_apply(struct rte_eth_dev *dev,
6046 struct rte_flow *flow,
6047 struct rte_flow_error *error)
6051 flow_d_shared_lock(dev);
6052 ret = flow_dv_apply(dev, flow, error);
6053 flow_d_shared_unlock(dev);
6058 * Mutex-protected thunk to flow_dv_remove().
6061 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
6063 flow_d_shared_lock(dev);
6064 flow_dv_remove(dev, flow);
6065 flow_d_shared_unlock(dev);
6069 * Mutex-protected thunk to flow_dv_destroy().
6072 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
6074 flow_d_shared_lock(dev);
6075 flow_dv_destroy(dev, flow);
6076 flow_d_shared_unlock(dev);
6079 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
6080 .validate = flow_dv_validate,
6081 .prepare = flow_dv_prepare,
6082 .translate = flow_d_translate,
6083 .apply = flow_d_apply,
6084 .remove = flow_d_remove,
6085 .destroy = flow_d_destroy,
6086 .query = flow_dv_query,
6089 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */