1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
31 #include "mlx5_defs.h"
32 #include "mlx5_glue.h"
33 #include "mlx5_flow.h"
35 #include "mlx5_rxtx.h"
37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
43 #ifndef HAVE_MLX5DV_DR_ESWITCH
44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
49 #ifndef HAVE_MLX5DV_DR
50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
53 /* VLAN header definitions */
54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
73 * Initialize flow attributes structure according to flow items' types.
76 * Pointer to item specification.
78 * Pointer to flow attributes structure.
81 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
83 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
85 case RTE_FLOW_ITEM_TYPE_IPV4:
88 case RTE_FLOW_ITEM_TYPE_IPV6:
91 case RTE_FLOW_ITEM_TYPE_UDP:
94 case RTE_FLOW_ITEM_TYPE_TCP:
104 struct field_modify_info {
105 uint32_t size; /* Size of field in protocol header, in bytes. */
106 uint32_t offset; /* Offset of field in protocol header, in bytes. */
107 enum mlx5_modification_field id;
110 struct field_modify_info modify_eth[] = {
111 {4, 0, MLX5_MODI_OUT_DMAC_47_16},
112 {2, 4, MLX5_MODI_OUT_DMAC_15_0},
113 {4, 6, MLX5_MODI_OUT_SMAC_47_16},
114 {2, 10, MLX5_MODI_OUT_SMAC_15_0},
118 struct field_modify_info modify_ipv4[] = {
119 {1, 8, MLX5_MODI_OUT_IPV4_TTL},
120 {4, 12, MLX5_MODI_OUT_SIPV4},
121 {4, 16, MLX5_MODI_OUT_DIPV4},
125 struct field_modify_info modify_ipv6[] = {
126 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
127 {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
128 {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
129 {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
130 {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
131 {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
132 {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
133 {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
134 {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
138 struct field_modify_info modify_udp[] = {
139 {2, 0, MLX5_MODI_OUT_UDP_SPORT},
140 {2, 2, MLX5_MODI_OUT_UDP_DPORT},
144 struct field_modify_info modify_tcp[] = {
145 {2, 0, MLX5_MODI_OUT_TCP_SPORT},
146 {2, 2, MLX5_MODI_OUT_TCP_DPORT},
147 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
148 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
153 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
154 uint8_t next_protocol, uint64_t *item_flags,
157 assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
158 item->type == RTE_FLOW_ITEM_TYPE_IPV6);
159 if (next_protocol == IPPROTO_IPIP) {
160 *item_flags |= MLX5_FLOW_LAYER_IPIP;
163 if (next_protocol == IPPROTO_IPV6) {
164 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
170 * Acquire the synchronizing object to protect multithreaded access
171 * to shared dv context. Lock occurs only if context is actually
172 * shared, i.e. we have multiport IB device and representors are
176 * Pointer to the rte_eth_dev structure.
179 flow_d_shared_lock(struct rte_eth_dev *dev)
181 struct mlx5_priv *priv = dev->data->dev_private;
182 struct mlx5_ibv_shared *sh = priv->sh;
184 if (sh->dv_refcnt > 1) {
187 ret = pthread_mutex_lock(&sh->dv_mutex);
194 flow_d_shared_unlock(struct rte_eth_dev *dev)
196 struct mlx5_priv *priv = dev->data->dev_private;
197 struct mlx5_ibv_shared *sh = priv->sh;
199 if (sh->dv_refcnt > 1) {
202 ret = pthread_mutex_unlock(&sh->dv_mutex);
209 * Convert modify-header action to DV specification.
212 * Pointer to item specification.
214 * Pointer to field modification information.
215 * @param[in,out] resource
216 * Pointer to the modify-header resource.
218 * Type of modification.
220 * Pointer to the error structure.
223 * 0 on success, a negative errno value otherwise and rte_errno is set.
226 flow_dv_convert_modify_action(struct rte_flow_item *item,
227 struct field_modify_info *field,
228 struct mlx5_flow_dv_modify_hdr_resource *resource,
230 struct rte_flow_error *error)
232 uint32_t i = resource->actions_num;
233 struct mlx5_modification_cmd *actions = resource->actions;
234 const uint8_t *spec = item->spec;
235 const uint8_t *mask = item->mask;
238 while (field->size) {
240 /* Generate modify command for each mask segment. */
241 memcpy(&set, &mask[field->offset], field->size);
243 if (i >= MLX5_MODIFY_NUM)
244 return rte_flow_error_set(error, EINVAL,
245 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
246 "too many items to modify");
247 actions[i].action_type = type;
248 actions[i].field = field->id;
249 actions[i].length = field->size ==
250 4 ? 0 : field->size * 8;
251 rte_memcpy(&actions[i].data[4 - field->size],
252 &spec[field->offset], field->size);
253 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
256 if (resource->actions_num != i)
257 resource->actions_num = i;
260 if (!resource->actions_num)
261 return rte_flow_error_set(error, EINVAL,
262 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
263 "invalid modification flow item");
268 * Convert modify-header set IPv4 address action to DV specification.
270 * @param[in,out] resource
271 * Pointer to the modify-header resource.
273 * Pointer to action specification.
275 * Pointer to the error structure.
278 * 0 on success, a negative errno value otherwise and rte_errno is set.
281 flow_dv_convert_action_modify_ipv4
282 (struct mlx5_flow_dv_modify_hdr_resource *resource,
283 const struct rte_flow_action *action,
284 struct rte_flow_error *error)
286 const struct rte_flow_action_set_ipv4 *conf =
287 (const struct rte_flow_action_set_ipv4 *)(action->conf);
288 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
289 struct rte_flow_item_ipv4 ipv4;
290 struct rte_flow_item_ipv4 ipv4_mask;
292 memset(&ipv4, 0, sizeof(ipv4));
293 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
294 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
295 ipv4.hdr.src_addr = conf->ipv4_addr;
296 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
298 ipv4.hdr.dst_addr = conf->ipv4_addr;
299 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
302 item.mask = &ipv4_mask;
303 return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
304 MLX5_MODIFICATION_TYPE_SET, error);
308 * Convert modify-header set IPv6 address action to DV specification.
310 * @param[in,out] resource
311 * Pointer to the modify-header resource.
313 * Pointer to action specification.
315 * Pointer to the error structure.
318 * 0 on success, a negative errno value otherwise and rte_errno is set.
321 flow_dv_convert_action_modify_ipv6
322 (struct mlx5_flow_dv_modify_hdr_resource *resource,
323 const struct rte_flow_action *action,
324 struct rte_flow_error *error)
326 const struct rte_flow_action_set_ipv6 *conf =
327 (const struct rte_flow_action_set_ipv6 *)(action->conf);
328 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
329 struct rte_flow_item_ipv6 ipv6;
330 struct rte_flow_item_ipv6 ipv6_mask;
332 memset(&ipv6, 0, sizeof(ipv6));
333 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
334 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
335 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
336 sizeof(ipv6.hdr.src_addr));
337 memcpy(&ipv6_mask.hdr.src_addr,
338 &rte_flow_item_ipv6_mask.hdr.src_addr,
339 sizeof(ipv6.hdr.src_addr));
341 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
342 sizeof(ipv6.hdr.dst_addr));
343 memcpy(&ipv6_mask.hdr.dst_addr,
344 &rte_flow_item_ipv6_mask.hdr.dst_addr,
345 sizeof(ipv6.hdr.dst_addr));
348 item.mask = &ipv6_mask;
349 return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
350 MLX5_MODIFICATION_TYPE_SET, error);
354 * Convert modify-header set MAC address action to DV specification.
356 * @param[in,out] resource
357 * Pointer to the modify-header resource.
359 * Pointer to action specification.
361 * Pointer to the error structure.
364 * 0 on success, a negative errno value otherwise and rte_errno is set.
367 flow_dv_convert_action_modify_mac
368 (struct mlx5_flow_dv_modify_hdr_resource *resource,
369 const struct rte_flow_action *action,
370 struct rte_flow_error *error)
372 const struct rte_flow_action_set_mac *conf =
373 (const struct rte_flow_action_set_mac *)(action->conf);
374 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
375 struct rte_flow_item_eth eth;
376 struct rte_flow_item_eth eth_mask;
378 memset(ð, 0, sizeof(eth));
379 memset(ð_mask, 0, sizeof(eth_mask));
380 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
381 memcpy(ð.src.addr_bytes, &conf->mac_addr,
382 sizeof(eth.src.addr_bytes));
383 memcpy(ð_mask.src.addr_bytes,
384 &rte_flow_item_eth_mask.src.addr_bytes,
385 sizeof(eth_mask.src.addr_bytes));
387 memcpy(ð.dst.addr_bytes, &conf->mac_addr,
388 sizeof(eth.dst.addr_bytes));
389 memcpy(ð_mask.dst.addr_bytes,
390 &rte_flow_item_eth_mask.dst.addr_bytes,
391 sizeof(eth_mask.dst.addr_bytes));
394 item.mask = ð_mask;
395 return flow_dv_convert_modify_action(&item, modify_eth, resource,
396 MLX5_MODIFICATION_TYPE_SET, error);
400 * Convert modify-header set TP action to DV specification.
402 * @param[in,out] resource
403 * Pointer to the modify-header resource.
405 * Pointer to action specification.
407 * Pointer to rte_flow_item objects list.
409 * Pointer to flow attributes structure.
411 * Pointer to the error structure.
414 * 0 on success, a negative errno value otherwise and rte_errno is set.
417 flow_dv_convert_action_modify_tp
418 (struct mlx5_flow_dv_modify_hdr_resource *resource,
419 const struct rte_flow_action *action,
420 const struct rte_flow_item *items,
421 union flow_dv_attr *attr,
422 struct rte_flow_error *error)
424 const struct rte_flow_action_set_tp *conf =
425 (const struct rte_flow_action_set_tp *)(action->conf);
426 struct rte_flow_item item;
427 struct rte_flow_item_udp udp;
428 struct rte_flow_item_udp udp_mask;
429 struct rte_flow_item_tcp tcp;
430 struct rte_flow_item_tcp tcp_mask;
431 struct field_modify_info *field;
434 flow_dv_attr_init(items, attr);
436 memset(&udp, 0, sizeof(udp));
437 memset(&udp_mask, 0, sizeof(udp_mask));
438 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
439 udp.hdr.src_port = conf->port;
440 udp_mask.hdr.src_port =
441 rte_flow_item_udp_mask.hdr.src_port;
443 udp.hdr.dst_port = conf->port;
444 udp_mask.hdr.dst_port =
445 rte_flow_item_udp_mask.hdr.dst_port;
447 item.type = RTE_FLOW_ITEM_TYPE_UDP;
449 item.mask = &udp_mask;
453 memset(&tcp, 0, sizeof(tcp));
454 memset(&tcp_mask, 0, sizeof(tcp_mask));
455 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
456 tcp.hdr.src_port = conf->port;
457 tcp_mask.hdr.src_port =
458 rte_flow_item_tcp_mask.hdr.src_port;
460 tcp.hdr.dst_port = conf->port;
461 tcp_mask.hdr.dst_port =
462 rte_flow_item_tcp_mask.hdr.dst_port;
464 item.type = RTE_FLOW_ITEM_TYPE_TCP;
466 item.mask = &tcp_mask;
469 return flow_dv_convert_modify_action(&item, field, resource,
470 MLX5_MODIFICATION_TYPE_SET, error);
474 * Convert modify-header set TTL action to DV specification.
476 * @param[in,out] resource
477 * Pointer to the modify-header resource.
479 * Pointer to action specification.
481 * Pointer to rte_flow_item objects list.
483 * Pointer to flow attributes structure.
485 * Pointer to the error structure.
488 * 0 on success, a negative errno value otherwise and rte_errno is set.
491 flow_dv_convert_action_modify_ttl
492 (struct mlx5_flow_dv_modify_hdr_resource *resource,
493 const struct rte_flow_action *action,
494 const struct rte_flow_item *items,
495 union flow_dv_attr *attr,
496 struct rte_flow_error *error)
498 const struct rte_flow_action_set_ttl *conf =
499 (const struct rte_flow_action_set_ttl *)(action->conf);
500 struct rte_flow_item item;
501 struct rte_flow_item_ipv4 ipv4;
502 struct rte_flow_item_ipv4 ipv4_mask;
503 struct rte_flow_item_ipv6 ipv6;
504 struct rte_flow_item_ipv6 ipv6_mask;
505 struct field_modify_info *field;
508 flow_dv_attr_init(items, attr);
510 memset(&ipv4, 0, sizeof(ipv4));
511 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
512 ipv4.hdr.time_to_live = conf->ttl_value;
513 ipv4_mask.hdr.time_to_live = 0xFF;
514 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
516 item.mask = &ipv4_mask;
520 memset(&ipv6, 0, sizeof(ipv6));
521 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
522 ipv6.hdr.hop_limits = conf->ttl_value;
523 ipv6_mask.hdr.hop_limits = 0xFF;
524 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
526 item.mask = &ipv6_mask;
529 return flow_dv_convert_modify_action(&item, field, resource,
530 MLX5_MODIFICATION_TYPE_SET, error);
534 * Convert modify-header decrement TTL action to DV specification.
536 * @param[in,out] resource
537 * Pointer to the modify-header resource.
539 * Pointer to action specification.
541 * Pointer to rte_flow_item objects list.
543 * Pointer to flow attributes structure.
545 * Pointer to the error structure.
548 * 0 on success, a negative errno value otherwise and rte_errno is set.
551 flow_dv_convert_action_modify_dec_ttl
552 (struct mlx5_flow_dv_modify_hdr_resource *resource,
553 const struct rte_flow_item *items,
554 union flow_dv_attr *attr,
555 struct rte_flow_error *error)
557 struct rte_flow_item item;
558 struct rte_flow_item_ipv4 ipv4;
559 struct rte_flow_item_ipv4 ipv4_mask;
560 struct rte_flow_item_ipv6 ipv6;
561 struct rte_flow_item_ipv6 ipv6_mask;
562 struct field_modify_info *field;
565 flow_dv_attr_init(items, attr);
567 memset(&ipv4, 0, sizeof(ipv4));
568 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
569 ipv4.hdr.time_to_live = 0xFF;
570 ipv4_mask.hdr.time_to_live = 0xFF;
571 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
573 item.mask = &ipv4_mask;
577 memset(&ipv6, 0, sizeof(ipv6));
578 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
579 ipv6.hdr.hop_limits = 0xFF;
580 ipv6_mask.hdr.hop_limits = 0xFF;
581 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
583 item.mask = &ipv6_mask;
586 return flow_dv_convert_modify_action(&item, field, resource,
587 MLX5_MODIFICATION_TYPE_ADD, error);
591 * Convert modify-header increment/decrement TCP Sequence number
592 * to DV specification.
594 * @param[in,out] resource
595 * Pointer to the modify-header resource.
597 * Pointer to action specification.
599 * Pointer to the error structure.
602 * 0 on success, a negative errno value otherwise and rte_errno is set.
605 flow_dv_convert_action_modify_tcp_seq
606 (struct mlx5_flow_dv_modify_hdr_resource *resource,
607 const struct rte_flow_action *action,
608 struct rte_flow_error *error)
610 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
611 uint64_t value = rte_be_to_cpu_32(*conf);
612 struct rte_flow_item item;
613 struct rte_flow_item_tcp tcp;
614 struct rte_flow_item_tcp tcp_mask;
616 memset(&tcp, 0, sizeof(tcp));
617 memset(&tcp_mask, 0, sizeof(tcp_mask));
618 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
620 * The HW has no decrement operation, only increment operation.
621 * To simulate decrement X from Y using increment operation
622 * we need to add UINT32_MAX X times to Y.
623 * Each adding of UINT32_MAX decrements Y by 1.
626 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
627 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
628 item.type = RTE_FLOW_ITEM_TYPE_TCP;
630 item.mask = &tcp_mask;
631 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
632 MLX5_MODIFICATION_TYPE_ADD, error);
636 * Convert modify-header increment/decrement TCP Acknowledgment number
637 * to DV specification.
639 * @param[in,out] resource
640 * Pointer to the modify-header resource.
642 * Pointer to action specification.
644 * Pointer to the error structure.
647 * 0 on success, a negative errno value otherwise and rte_errno is set.
650 flow_dv_convert_action_modify_tcp_ack
651 (struct mlx5_flow_dv_modify_hdr_resource *resource,
652 const struct rte_flow_action *action,
653 struct rte_flow_error *error)
655 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
656 uint64_t value = rte_be_to_cpu_32(*conf);
657 struct rte_flow_item item;
658 struct rte_flow_item_tcp tcp;
659 struct rte_flow_item_tcp tcp_mask;
661 memset(&tcp, 0, sizeof(tcp));
662 memset(&tcp_mask, 0, sizeof(tcp_mask));
663 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
665 * The HW has no decrement operation, only increment operation.
666 * To simulate decrement X from Y using increment operation
667 * we need to add UINT32_MAX X times to Y.
668 * Each adding of UINT32_MAX decrements Y by 1.
671 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
672 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
673 item.type = RTE_FLOW_ITEM_TYPE_TCP;
675 item.mask = &tcp_mask;
676 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
677 MLX5_MODIFICATION_TYPE_ADD, error);
681 * Validate META item.
684 * Pointer to the rte_eth_dev structure.
686 * Item specification.
688 * Attributes of flow that includes this item.
690 * Pointer to error structure.
693 * 0 on success, a negative errno value otherwise and rte_errno is set.
696 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
697 const struct rte_flow_item *item,
698 const struct rte_flow_attr *attr,
699 struct rte_flow_error *error)
701 const struct rte_flow_item_meta *spec = item->spec;
702 const struct rte_flow_item_meta *mask = item->mask;
703 const struct rte_flow_item_meta nic_mask = {
704 .data = RTE_BE32(UINT32_MAX)
707 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
709 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
710 return rte_flow_error_set(error, EPERM,
711 RTE_FLOW_ERROR_TYPE_ITEM,
713 "match on metadata offload "
714 "configuration is off for this port");
716 return rte_flow_error_set(error, EINVAL,
717 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
719 "data cannot be empty");
721 return rte_flow_error_set(error, EINVAL,
722 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
724 "data cannot be zero");
726 mask = &rte_flow_item_meta_mask;
727 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
728 (const uint8_t *)&nic_mask,
729 sizeof(struct rte_flow_item_meta),
734 return rte_flow_error_set(error, ENOTSUP,
735 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
737 "pattern not supported for ingress");
742 * Validate vport item.
745 * Pointer to the rte_eth_dev structure.
747 * Item specification.
749 * Attributes of flow that includes this item.
750 * @param[in] item_flags
751 * Bit-fields that holds the items detected until now.
753 * Pointer to error structure.
756 * 0 on success, a negative errno value otherwise and rte_errno is set.
759 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
760 const struct rte_flow_item *item,
761 const struct rte_flow_attr *attr,
763 struct rte_flow_error *error)
765 const struct rte_flow_item_port_id *spec = item->spec;
766 const struct rte_flow_item_port_id *mask = item->mask;
767 const struct rte_flow_item_port_id switch_mask = {
770 uint16_t esw_domain_id;
771 uint16_t item_port_esw_domain_id;
775 return rte_flow_error_set(error, EINVAL,
776 RTE_FLOW_ERROR_TYPE_ITEM,
778 "match on port id is valid only"
779 " when transfer flag is enabled");
780 if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
781 return rte_flow_error_set(error, ENOTSUP,
782 RTE_FLOW_ERROR_TYPE_ITEM, item,
783 "multiple source ports are not"
787 if (mask->id != 0xffffffff)
788 return rte_flow_error_set(error, ENOTSUP,
789 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
791 "no support for partial mask on"
793 ret = mlx5_flow_item_acceptable
794 (item, (const uint8_t *)mask,
795 (const uint8_t *)&rte_flow_item_port_id_mask,
796 sizeof(struct rte_flow_item_port_id),
802 ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
805 return rte_flow_error_set(error, -ret,
806 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
807 "failed to obtain E-Switch info for"
809 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
810 &esw_domain_id, NULL);
812 return rte_flow_error_set(error, -ret,
813 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
815 "failed to obtain E-Switch info");
816 if (item_port_esw_domain_id != esw_domain_id)
817 return rte_flow_error_set(error, -ret,
818 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
819 "cannot match on a port from a"
820 " different E-Switch");
825 * Validate the pop VLAN action.
828 * Pointer to the rte_eth_dev structure.
829 * @param[in] action_flags
830 * Holds the actions detected until now.
832 * Pointer to the pop vlan action.
833 * @param[in] item_flags
834 * The items found in this flow rule.
836 * Pointer to flow attributes.
838 * Pointer to error structure.
841 * 0 on success, a negative errno value otherwise and rte_errno is set.
844 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
845 uint64_t action_flags,
846 const struct rte_flow_action *action,
848 const struct rte_flow_attr *attr,
849 struct rte_flow_error *error)
851 struct mlx5_priv *priv = dev->data->dev_private;
855 if (!priv->sh->pop_vlan_action)
856 return rte_flow_error_set(error, ENOTSUP,
857 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
859 "pop vlan action is not supported");
861 * Check for inconsistencies:
862 * fail strip_vlan in a flow that matches packets without VLAN tags.
863 * fail strip_vlan in a flow that matches packets without explicitly a
864 * matching on VLAN tag ?
866 if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN)
867 return rte_flow_error_set(error, ENOTSUP,
868 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
870 "no support for multiple vlan pop "
872 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
873 return rte_flow_error_set(error, ENOTSUP,
874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
876 "cannot pop vlan without a "
877 "match on (outer) vlan in the flow");
882 * Get VLAN default info from vlan match info.
885 * Pointer to the rte_eth_dev structure.
887 * the list of item specifications.
889 * pointer VLAN info to fill to.
891 * Pointer to error structure.
894 * 0 on success, a negative errno value otherwise and rte_errno is set.
897 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
898 struct rte_vlan_hdr *vlan)
900 const struct rte_flow_item_vlan nic_mask = {
901 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
902 MLX5DV_FLOW_VLAN_VID_MASK),
903 .inner_type = RTE_BE16(0xffff),
908 for (; items->type != RTE_FLOW_ITEM_TYPE_END &&
909 items->type != RTE_FLOW_ITEM_TYPE_VLAN; items++)
911 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
912 const struct rte_flow_item_vlan *vlan_m = items->mask;
913 const struct rte_flow_item_vlan *vlan_v = items->spec;
917 /* Only full match values are accepted */
918 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
919 MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
920 vlan->vlan_tci &= MLX5DV_FLOW_VLAN_PCP_MASK;
922 rte_be_to_cpu_16(vlan_v->tci &
923 MLX5DV_FLOW_VLAN_PCP_MASK_BE);
925 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
926 MLX5DV_FLOW_VLAN_VID_MASK_BE) {
927 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
929 rte_be_to_cpu_16(vlan_v->tci &
930 MLX5DV_FLOW_VLAN_VID_MASK_BE);
932 if (vlan_m->inner_type == nic_mask.inner_type)
933 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
939 * Validate the push VLAN action.
941 * @param[in] action_flags
942 * Holds the actions detected until now.
944 * Pointer to the encap action.
946 * Pointer to flow attributes
948 * Pointer to error structure.
951 * 0 on success, a negative errno value otherwise and rte_errno is set.
954 flow_dv_validate_action_push_vlan(uint64_t action_flags,
955 const struct rte_flow_action *action,
956 const struct rte_flow_attr *attr,
957 struct rte_flow_error *error)
959 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
961 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
962 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
963 return rte_flow_error_set(error, EINVAL,
964 RTE_FLOW_ERROR_TYPE_ACTION, action,
965 "invalid vlan ethertype");
967 (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN))
968 return rte_flow_error_set(error, ENOTSUP,
969 RTE_FLOW_ERROR_TYPE_ACTION, action,
970 "no support for multiple VLAN "
977 * Validate the set VLAN PCP.
979 * @param[in] action_flags
980 * Holds the actions detected until now.
982 * Pointer to the list of actions remaining in the flow rule.
984 * Pointer to flow attributes
986 * Pointer to error structure.
989 * 0 on success, a negative errno value otherwise and rte_errno is set.
992 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
993 const struct rte_flow_action actions[],
994 struct rte_flow_error *error)
996 const struct rte_flow_action *action = actions;
997 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
999 if (conf->vlan_pcp > 7)
1000 return rte_flow_error_set(error, EINVAL,
1001 RTE_FLOW_ERROR_TYPE_ACTION, action,
1002 "VLAN PCP value is too big");
1003 if (mlx5_flow_find_action(actions,
1004 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) == NULL)
1005 return rte_flow_error_set(error, ENOTSUP,
1006 RTE_FLOW_ERROR_TYPE_ACTION, action,
1007 "set VLAN PCP can only be used "
1008 "with push VLAN action");
1009 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
1010 return rte_flow_error_set(error, ENOTSUP,
1011 RTE_FLOW_ERROR_TYPE_ACTION, action,
1012 "set VLAN PCP action must precede "
1013 "the push VLAN action");
1018 * Validate the set VLAN VID.
1020 * @param[in] action_flags
1021 * Holds the actions detected until now.
1022 * @param[in] actions
1023 * Pointer to the list of actions remaining in the flow rule.
1025 * Pointer to flow attributes
1027 * Pointer to error structure.
1030 * 0 on success, a negative errno value otherwise and rte_errno is set.
1033 flow_dv_validate_action_set_vlan_vid(uint64_t action_flags,
1034 const struct rte_flow_action actions[],
1035 struct rte_flow_error *error)
1037 const struct rte_flow_action *action = actions;
1038 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
1040 if (conf->vlan_vid > RTE_BE16(0xFFE))
1041 return rte_flow_error_set(error, EINVAL,
1042 RTE_FLOW_ERROR_TYPE_ACTION, action,
1043 "VLAN VID value is too big");
1044 if (mlx5_flow_find_action(actions,
1045 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) == NULL)
1046 return rte_flow_error_set(error, ENOTSUP,
1047 RTE_FLOW_ERROR_TYPE_ACTION, action,
1048 "set VLAN VID can only be used "
1049 "with push VLAN action");
1050 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
1051 return rte_flow_error_set(error, ENOTSUP,
1052 RTE_FLOW_ERROR_TYPE_ACTION, action,
1053 "set VLAN VID action must precede "
1054 "the push VLAN action");
1059 * Validate count action.
1064 * Pointer to error structure.
1067 * 0 on success, a negative errno value otherwise and rte_errno is set.
1070 flow_dv_validate_action_count(struct rte_eth_dev *dev,
1071 struct rte_flow_error *error)
1073 struct mlx5_priv *priv = dev->data->dev_private;
1075 if (!priv->config.devx)
1077 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
1081 return rte_flow_error_set
1083 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1085 "count action not supported");
1089 * Validate the L2 encap action.
1091 * @param[in] action_flags
1092 * Holds the actions detected until now.
1094 * Pointer to the encap action.
1096 * Pointer to flow attributes
1098 * Pointer to error structure.
1101 * 0 on success, a negative errno value otherwise and rte_errno is set.
1104 flow_dv_validate_action_l2_encap(uint64_t action_flags,
1105 const struct rte_flow_action *action,
1106 const struct rte_flow_attr *attr,
1107 struct rte_flow_error *error)
1109 if (!(action->conf))
1110 return rte_flow_error_set(error, EINVAL,
1111 RTE_FLOW_ERROR_TYPE_ACTION, action,
1112 "configuration cannot be null");
1113 if (action_flags & MLX5_FLOW_ACTION_DROP)
1114 return rte_flow_error_set(error, EINVAL,
1115 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1116 "can't drop and encap in same flow");
1117 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1118 return rte_flow_error_set(error, EINVAL,
1119 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1120 "can only have a single encap or"
1121 " decap action in a flow");
1122 if (!attr->transfer && attr->ingress)
1123 return rte_flow_error_set(error, ENOTSUP,
1124 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1126 "encap action not supported for "
1132 * Validate the L2 decap action.
1134 * @param[in] action_flags
1135 * Holds the actions detected until now.
1137 * Pointer to flow attributes
1139 * Pointer to error structure.
1142 * 0 on success, a negative errno value otherwise and rte_errno is set.
1145 flow_dv_validate_action_l2_decap(uint64_t action_flags,
1146 const struct rte_flow_attr *attr,
1147 struct rte_flow_error *error)
1149 if (action_flags & MLX5_FLOW_ACTION_DROP)
1150 return rte_flow_error_set(error, EINVAL,
1151 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1152 "can't drop and decap in same flow");
1153 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1154 return rte_flow_error_set(error, EINVAL,
1155 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1156 "can only have a single encap or"
1157 " decap action in a flow");
1158 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1159 return rte_flow_error_set(error, EINVAL,
1160 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1161 "can't have decap action after"
1164 return rte_flow_error_set(error, ENOTSUP,
1165 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1167 "decap action not supported for "
1173 * Validate the raw encap action.
1175 * @param[in] action_flags
1176 * Holds the actions detected until now.
1178 * Pointer to the encap action.
1180 * Pointer to flow attributes
1182 * Pointer to error structure.
1185 * 0 on success, a negative errno value otherwise and rte_errno is set.
1188 flow_dv_validate_action_raw_encap(uint64_t action_flags,
1189 const struct rte_flow_action *action,
1190 const struct rte_flow_attr *attr,
1191 struct rte_flow_error *error)
1193 const struct rte_flow_action_raw_encap *raw_encap =
1194 (const struct rte_flow_action_raw_encap *)action->conf;
1195 if (!(action->conf))
1196 return rte_flow_error_set(error, EINVAL,
1197 RTE_FLOW_ERROR_TYPE_ACTION, action,
1198 "configuration cannot be null");
1199 if (action_flags & MLX5_FLOW_ACTION_DROP)
1200 return rte_flow_error_set(error, EINVAL,
1201 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1202 "can't drop and encap in same flow");
1203 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1204 return rte_flow_error_set(error, EINVAL,
1205 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1206 "can only have a single encap"
1207 " action in a flow");
1208 /* encap without preceding decap is not supported for ingress */
1209 if (!attr->transfer && attr->ingress &&
1210 !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
1211 return rte_flow_error_set(error, ENOTSUP,
1212 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1214 "encap action not supported for "
1216 if (!raw_encap->size || !raw_encap->data)
1217 return rte_flow_error_set(error, EINVAL,
1218 RTE_FLOW_ERROR_TYPE_ACTION, action,
1219 "raw encap data cannot be empty");
1224 * Validate the raw decap action.
1226 * @param[in] action_flags
1227 * Holds the actions detected until now.
1229 * Pointer to the encap action.
1231 * Pointer to flow attributes
1233 * Pointer to error structure.
1236 * 0 on success, a negative errno value otherwise and rte_errno is set.
1239 flow_dv_validate_action_raw_decap(uint64_t action_flags,
1240 const struct rte_flow_action *action,
1241 const struct rte_flow_attr *attr,
1242 struct rte_flow_error *error)
1244 if (action_flags & MLX5_FLOW_ACTION_DROP)
1245 return rte_flow_error_set(error, EINVAL,
1246 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1247 "can't drop and decap in same flow");
1248 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1249 return rte_flow_error_set(error, EINVAL,
1250 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1251 "can't have encap action before"
1253 if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1254 return rte_flow_error_set(error, EINVAL,
1255 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1256 "can only have a single decap"
1257 " action in a flow");
1258 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1259 return rte_flow_error_set(error, EINVAL,
1260 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1261 "can't have decap action after"
1263 /* decap action is valid on egress only if it is followed by encap */
1265 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1266 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1269 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1270 return rte_flow_error_set
1272 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1273 NULL, "decap action not supported"
1280 * Find existing encap/decap resource or create and register a new one.
1282 * @param dev[in, out]
1283 * Pointer to rte_eth_dev structure.
1284 * @param[in, out] resource
1285 * Pointer to encap/decap resource.
1286 * @parm[in, out] dev_flow
1287 * Pointer to the dev_flow.
1289 * pointer to error structure.
1292 * 0 on success otherwise -errno and errno is set.
1295 flow_dv_encap_decap_resource_register
1296 (struct rte_eth_dev *dev,
1297 struct mlx5_flow_dv_encap_decap_resource *resource,
1298 struct mlx5_flow *dev_flow,
1299 struct rte_flow_error *error)
1301 struct mlx5_priv *priv = dev->data->dev_private;
1302 struct mlx5_ibv_shared *sh = priv->sh;
1303 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1304 struct rte_flow *flow = dev_flow->flow;
1305 struct mlx5dv_dr_domain *domain;
1307 resource->flags = flow->group ? 0 : 1;
1308 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1309 domain = sh->fdb_domain;
1310 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1311 domain = sh->rx_domain;
1313 domain = sh->tx_domain;
1315 /* Lookup a matching resource from cache. */
1316 LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1317 if (resource->reformat_type == cache_resource->reformat_type &&
1318 resource->ft_type == cache_resource->ft_type &&
1319 resource->flags == cache_resource->flags &&
1320 resource->size == cache_resource->size &&
1321 !memcmp((const void *)resource->buf,
1322 (const void *)cache_resource->buf,
1324 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1325 (void *)cache_resource,
1326 rte_atomic32_read(&cache_resource->refcnt));
1327 rte_atomic32_inc(&cache_resource->refcnt);
1328 dev_flow->dv.encap_decap = cache_resource;
1332 /* Register new encap/decap resource. */
1333 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1334 if (!cache_resource)
1335 return rte_flow_error_set(error, ENOMEM,
1336 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1337 "cannot allocate resource memory");
1338 *cache_resource = *resource;
1339 cache_resource->verbs_action =
1340 mlx5_glue->dv_create_flow_action_packet_reformat
1341 (sh->ctx, cache_resource->reformat_type,
1342 cache_resource->ft_type, domain, cache_resource->flags,
1343 cache_resource->size,
1344 (cache_resource->size ? cache_resource->buf : NULL));
1345 if (!cache_resource->verbs_action) {
1346 rte_free(cache_resource);
1347 return rte_flow_error_set(error, ENOMEM,
1348 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1349 NULL, "cannot create action");
1351 rte_atomic32_init(&cache_resource->refcnt);
1352 rte_atomic32_inc(&cache_resource->refcnt);
1353 LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1354 dev_flow->dv.encap_decap = cache_resource;
1355 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1356 (void *)cache_resource,
1357 rte_atomic32_read(&cache_resource->refcnt));
1362 * Find existing table jump resource or create and register a new one.
1364 * @param dev[in, out]
1365 * Pointer to rte_eth_dev structure.
1366 * @param[in, out] resource
1367 * Pointer to jump table resource.
1368 * @parm[in, out] dev_flow
1369 * Pointer to the dev_flow.
1371 * pointer to error structure.
1374 * 0 on success otherwise -errno and errno is set.
1377 flow_dv_jump_tbl_resource_register
1378 (struct rte_eth_dev *dev,
1379 struct mlx5_flow_dv_jump_tbl_resource *resource,
1380 struct mlx5_flow *dev_flow,
1381 struct rte_flow_error *error)
1383 struct mlx5_priv *priv = dev->data->dev_private;
1384 struct mlx5_ibv_shared *sh = priv->sh;
1385 struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1387 /* Lookup a matching resource from cache. */
1388 LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1389 if (resource->tbl == cache_resource->tbl) {
1390 DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1391 (void *)cache_resource,
1392 rte_atomic32_read(&cache_resource->refcnt));
1393 rte_atomic32_inc(&cache_resource->refcnt);
1394 dev_flow->dv.jump = cache_resource;
1398 /* Register new jump table resource. */
1399 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1400 if (!cache_resource)
1401 return rte_flow_error_set(error, ENOMEM,
1402 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1403 "cannot allocate resource memory");
1404 *cache_resource = *resource;
1405 cache_resource->action =
1406 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1407 (resource->tbl->obj);
1408 if (!cache_resource->action) {
1409 rte_free(cache_resource);
1410 return rte_flow_error_set(error, ENOMEM,
1411 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1412 NULL, "cannot create action");
1414 rte_atomic32_init(&cache_resource->refcnt);
1415 rte_atomic32_inc(&cache_resource->refcnt);
1416 LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1417 dev_flow->dv.jump = cache_resource;
1418 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
1419 (void *)cache_resource,
1420 rte_atomic32_read(&cache_resource->refcnt));
1425 * Find existing table port ID resource or create and register a new one.
1427 * @param dev[in, out]
1428 * Pointer to rte_eth_dev structure.
1429 * @param[in, out] resource
1430 * Pointer to port ID action resource.
1431 * @parm[in, out] dev_flow
1432 * Pointer to the dev_flow.
1434 * pointer to error structure.
1437 * 0 on success otherwise -errno and errno is set.
1440 flow_dv_port_id_action_resource_register
1441 (struct rte_eth_dev *dev,
1442 struct mlx5_flow_dv_port_id_action_resource *resource,
1443 struct mlx5_flow *dev_flow,
1444 struct rte_flow_error *error)
1446 struct mlx5_priv *priv = dev->data->dev_private;
1447 struct mlx5_ibv_shared *sh = priv->sh;
1448 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1450 /* Lookup a matching resource from cache. */
1451 LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1452 if (resource->port_id == cache_resource->port_id) {
1453 DRV_LOG(DEBUG, "port id action resource resource %p: "
1455 (void *)cache_resource,
1456 rte_atomic32_read(&cache_resource->refcnt));
1457 rte_atomic32_inc(&cache_resource->refcnt);
1458 dev_flow->dv.port_id_action = cache_resource;
1462 /* Register new port id action resource. */
1463 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1464 if (!cache_resource)
1465 return rte_flow_error_set(error, ENOMEM,
1466 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1467 "cannot allocate resource memory");
1468 *cache_resource = *resource;
1469 cache_resource->action =
1470 mlx5_glue->dr_create_flow_action_dest_vport
1471 (priv->sh->fdb_domain, resource->port_id);
1472 if (!cache_resource->action) {
1473 rte_free(cache_resource);
1474 return rte_flow_error_set(error, ENOMEM,
1475 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1476 NULL, "cannot create action");
1478 rte_atomic32_init(&cache_resource->refcnt);
1479 rte_atomic32_inc(&cache_resource->refcnt);
1480 LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1481 dev_flow->dv.port_id_action = cache_resource;
1482 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1483 (void *)cache_resource,
1484 rte_atomic32_read(&cache_resource->refcnt));
1489 * Find existing push vlan resource or create and register a new one.
1491 * @param dev[in, out]
1492 * Pointer to rte_eth_dev structure.
1493 * @param[in, out] resource
1494 * Pointer to port ID action resource.
1495 * @parm[in, out] dev_flow
1496 * Pointer to the dev_flow.
1498 * pointer to error structure.
1501 * 0 on success otherwise -errno and errno is set.
1504 flow_dv_push_vlan_action_resource_register
1505 (struct rte_eth_dev *dev,
1506 struct mlx5_flow_dv_push_vlan_action_resource *resource,
1507 struct mlx5_flow *dev_flow,
1508 struct rte_flow_error *error)
1510 struct mlx5_priv *priv = dev->data->dev_private;
1511 struct mlx5_ibv_shared *sh = priv->sh;
1512 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
1513 struct mlx5dv_dr_domain *domain;
1515 /* Lookup a matching resource from cache. */
1516 LIST_FOREACH(cache_resource, &sh->push_vlan_action_list, next) {
1517 if (resource->vlan_tag == cache_resource->vlan_tag &&
1518 resource->ft_type == cache_resource->ft_type) {
1519 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
1521 (void *)cache_resource,
1522 rte_atomic32_read(&cache_resource->refcnt));
1523 rte_atomic32_inc(&cache_resource->refcnt);
1524 dev_flow->dv.push_vlan_res = cache_resource;
1528 /* Register new push_vlan action resource. */
1529 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1530 if (!cache_resource)
1531 return rte_flow_error_set(error, ENOMEM,
1532 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1533 "cannot allocate resource memory");
1534 *cache_resource = *resource;
1535 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1536 domain = sh->fdb_domain;
1537 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1538 domain = sh->rx_domain;
1540 domain = sh->tx_domain;
1541 cache_resource->action =
1542 mlx5_glue->dr_create_flow_action_push_vlan(domain,
1543 resource->vlan_tag);
1544 if (!cache_resource->action) {
1545 rte_free(cache_resource);
1546 return rte_flow_error_set(error, ENOMEM,
1547 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1548 NULL, "cannot create action");
1550 rte_atomic32_init(&cache_resource->refcnt);
1551 rte_atomic32_inc(&cache_resource->refcnt);
1552 LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next);
1553 dev_flow->dv.push_vlan_res = cache_resource;
1554 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
1555 (void *)cache_resource,
1556 rte_atomic32_read(&cache_resource->refcnt));
1560 * Get the size of specific rte_flow_item_type
1562 * @param[in] item_type
1563 * Tested rte_flow_item_type.
1566 * sizeof struct item_type, 0 if void or irrelevant.
1569 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1573 switch (item_type) {
1574 case RTE_FLOW_ITEM_TYPE_ETH:
1575 retval = sizeof(struct rte_flow_item_eth);
1577 case RTE_FLOW_ITEM_TYPE_VLAN:
1578 retval = sizeof(struct rte_flow_item_vlan);
1580 case RTE_FLOW_ITEM_TYPE_IPV4:
1581 retval = sizeof(struct rte_flow_item_ipv4);
1583 case RTE_FLOW_ITEM_TYPE_IPV6:
1584 retval = sizeof(struct rte_flow_item_ipv6);
1586 case RTE_FLOW_ITEM_TYPE_UDP:
1587 retval = sizeof(struct rte_flow_item_udp);
1589 case RTE_FLOW_ITEM_TYPE_TCP:
1590 retval = sizeof(struct rte_flow_item_tcp);
1592 case RTE_FLOW_ITEM_TYPE_VXLAN:
1593 retval = sizeof(struct rte_flow_item_vxlan);
1595 case RTE_FLOW_ITEM_TYPE_GRE:
1596 retval = sizeof(struct rte_flow_item_gre);
1598 case RTE_FLOW_ITEM_TYPE_NVGRE:
1599 retval = sizeof(struct rte_flow_item_nvgre);
1601 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1602 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1604 case RTE_FLOW_ITEM_TYPE_MPLS:
1605 retval = sizeof(struct rte_flow_item_mpls);
1607 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1615 #define MLX5_ENCAP_IPV4_VERSION 0x40
1616 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
1617 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
1618 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
1619 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
1620 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
1621 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
1624 * Convert the encap action data from list of rte_flow_item to raw buffer
1627 * Pointer to rte_flow_item objects list.
1629 * Pointer to the output buffer.
1631 * Pointer to the output buffer size.
1633 * Pointer to the error structure.
1636 * 0 on success, a negative errno value otherwise and rte_errno is set.
1639 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1640 size_t *size, struct rte_flow_error *error)
1642 struct rte_ether_hdr *eth = NULL;
1643 struct rte_vlan_hdr *vlan = NULL;
1644 struct rte_ipv4_hdr *ipv4 = NULL;
1645 struct rte_ipv6_hdr *ipv6 = NULL;
1646 struct rte_udp_hdr *udp = NULL;
1647 struct rte_vxlan_hdr *vxlan = NULL;
1648 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1649 struct rte_gre_hdr *gre = NULL;
1651 size_t temp_size = 0;
1654 return rte_flow_error_set(error, EINVAL,
1655 RTE_FLOW_ERROR_TYPE_ACTION,
1656 NULL, "invalid empty data");
1657 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1658 len = flow_dv_get_item_len(items->type);
1659 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1660 return rte_flow_error_set(error, EINVAL,
1661 RTE_FLOW_ERROR_TYPE_ACTION,
1662 (void *)items->type,
1663 "items total size is too big"
1664 " for encap action");
1665 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1666 switch (items->type) {
1667 case RTE_FLOW_ITEM_TYPE_ETH:
1668 eth = (struct rte_ether_hdr *)&buf[temp_size];
1670 case RTE_FLOW_ITEM_TYPE_VLAN:
1671 vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1673 return rte_flow_error_set(error, EINVAL,
1674 RTE_FLOW_ERROR_TYPE_ACTION,
1675 (void *)items->type,
1676 "eth header not found");
1677 if (!eth->ether_type)
1678 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1680 case RTE_FLOW_ITEM_TYPE_IPV4:
1681 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1683 return rte_flow_error_set(error, EINVAL,
1684 RTE_FLOW_ERROR_TYPE_ACTION,
1685 (void *)items->type,
1686 "neither eth nor vlan"
1688 if (vlan && !vlan->eth_proto)
1689 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1690 else if (eth && !eth->ether_type)
1691 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1692 if (!ipv4->version_ihl)
1693 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1694 MLX5_ENCAP_IPV4_IHL_MIN;
1695 if (!ipv4->time_to_live)
1696 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1698 case RTE_FLOW_ITEM_TYPE_IPV6:
1699 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1701 return rte_flow_error_set(error, EINVAL,
1702 RTE_FLOW_ERROR_TYPE_ACTION,
1703 (void *)items->type,
1704 "neither eth nor vlan"
1706 if (vlan && !vlan->eth_proto)
1707 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1708 else if (eth && !eth->ether_type)
1709 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1710 if (!ipv6->vtc_flow)
1712 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1713 if (!ipv6->hop_limits)
1714 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1716 case RTE_FLOW_ITEM_TYPE_UDP:
1717 udp = (struct rte_udp_hdr *)&buf[temp_size];
1719 return rte_flow_error_set(error, EINVAL,
1720 RTE_FLOW_ERROR_TYPE_ACTION,
1721 (void *)items->type,
1722 "ip header not found");
1723 if (ipv4 && !ipv4->next_proto_id)
1724 ipv4->next_proto_id = IPPROTO_UDP;
1725 else if (ipv6 && !ipv6->proto)
1726 ipv6->proto = IPPROTO_UDP;
1728 case RTE_FLOW_ITEM_TYPE_VXLAN:
1729 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1731 return rte_flow_error_set(error, EINVAL,
1732 RTE_FLOW_ERROR_TYPE_ACTION,
1733 (void *)items->type,
1734 "udp header not found");
1736 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1737 if (!vxlan->vx_flags)
1739 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1741 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1742 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1744 return rte_flow_error_set(error, EINVAL,
1745 RTE_FLOW_ERROR_TYPE_ACTION,
1746 (void *)items->type,
1747 "udp header not found");
1748 if (!vxlan_gpe->proto)
1749 return rte_flow_error_set(error, EINVAL,
1750 RTE_FLOW_ERROR_TYPE_ACTION,
1751 (void *)items->type,
1752 "next protocol not found");
1755 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1756 if (!vxlan_gpe->vx_flags)
1757 vxlan_gpe->vx_flags =
1758 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1760 case RTE_FLOW_ITEM_TYPE_GRE:
1761 case RTE_FLOW_ITEM_TYPE_NVGRE:
1762 gre = (struct rte_gre_hdr *)&buf[temp_size];
1764 return rte_flow_error_set(error, EINVAL,
1765 RTE_FLOW_ERROR_TYPE_ACTION,
1766 (void *)items->type,
1767 "next protocol not found");
1769 return rte_flow_error_set(error, EINVAL,
1770 RTE_FLOW_ERROR_TYPE_ACTION,
1771 (void *)items->type,
1772 "ip header not found");
1773 if (ipv4 && !ipv4->next_proto_id)
1774 ipv4->next_proto_id = IPPROTO_GRE;
1775 else if (ipv6 && !ipv6->proto)
1776 ipv6->proto = IPPROTO_GRE;
1778 case RTE_FLOW_ITEM_TYPE_VOID:
1781 return rte_flow_error_set(error, EINVAL,
1782 RTE_FLOW_ERROR_TYPE_ACTION,
1783 (void *)items->type,
1784 "unsupported item type");
1794 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1796 struct rte_ether_hdr *eth = NULL;
1797 struct rte_vlan_hdr *vlan = NULL;
1798 struct rte_ipv6_hdr *ipv6 = NULL;
1799 struct rte_udp_hdr *udp = NULL;
1803 eth = (struct rte_ether_hdr *)data;
1804 next_hdr = (char *)(eth + 1);
1805 proto = RTE_BE16(eth->ether_type);
1808 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1809 vlan = (struct rte_vlan_hdr *)next_hdr;
1810 proto = RTE_BE16(vlan->eth_proto);
1811 next_hdr += sizeof(struct rte_vlan_hdr);
1814 /* HW calculates IPv4 csum. no need to proceed */
1815 if (proto == RTE_ETHER_TYPE_IPV4)
1818 /* non IPv4/IPv6 header. not supported */
1819 if (proto != RTE_ETHER_TYPE_IPV6) {
1820 return rte_flow_error_set(error, ENOTSUP,
1821 RTE_FLOW_ERROR_TYPE_ACTION,
1822 NULL, "Cannot offload non IPv4/IPv6");
1825 ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1827 /* ignore non UDP */
1828 if (ipv6->proto != IPPROTO_UDP)
1831 udp = (struct rte_udp_hdr *)(ipv6 + 1);
1832 udp->dgram_cksum = 0;
1838 * Convert L2 encap action to DV specification.
1841 * Pointer to rte_eth_dev structure.
1843 * Pointer to action structure.
1844 * @param[in, out] dev_flow
1845 * Pointer to the mlx5_flow.
1846 * @param[in] transfer
1847 * Mark if the flow is E-Switch flow.
1849 * Pointer to the error structure.
1852 * 0 on success, a negative errno value otherwise and rte_errno is set.
1855 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1856 const struct rte_flow_action *action,
1857 struct mlx5_flow *dev_flow,
1859 struct rte_flow_error *error)
1861 const struct rte_flow_item *encap_data;
1862 const struct rte_flow_action_raw_encap *raw_encap_data;
1863 struct mlx5_flow_dv_encap_decap_resource res = {
1865 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1866 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1867 MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1870 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1872 (const struct rte_flow_action_raw_encap *)action->conf;
1873 res.size = raw_encap_data->size;
1874 memcpy(res.buf, raw_encap_data->data, res.size);
1875 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1878 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1880 ((const struct rte_flow_action_vxlan_encap *)
1881 action->conf)->definition;
1884 ((const struct rte_flow_action_nvgre_encap *)
1885 action->conf)->definition;
1886 if (flow_dv_convert_encap_data(encap_data, res.buf,
1890 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1891 return rte_flow_error_set(error, EINVAL,
1892 RTE_FLOW_ERROR_TYPE_ACTION,
1893 NULL, "can't create L2 encap action");
1898 * Convert L2 decap action to DV specification.
1901 * Pointer to rte_eth_dev structure.
1902 * @param[in, out] dev_flow
1903 * Pointer to the mlx5_flow.
1904 * @param[in] transfer
1905 * Mark if the flow is E-Switch flow.
1907 * Pointer to the error structure.
1910 * 0 on success, a negative errno value otherwise and rte_errno is set.
1913 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1914 struct mlx5_flow *dev_flow,
1916 struct rte_flow_error *error)
1918 struct mlx5_flow_dv_encap_decap_resource res = {
1921 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1922 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1923 MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1926 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1927 return rte_flow_error_set(error, EINVAL,
1928 RTE_FLOW_ERROR_TYPE_ACTION,
1929 NULL, "can't create L2 decap action");
1934 * Convert raw decap/encap (L3 tunnel) action to DV specification.
1937 * Pointer to rte_eth_dev structure.
1939 * Pointer to action structure.
1940 * @param[in, out] dev_flow
1941 * Pointer to the mlx5_flow.
1943 * Pointer to the flow attributes.
1945 * Pointer to the error structure.
1948 * 0 on success, a negative errno value otherwise and rte_errno is set.
1951 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1952 const struct rte_flow_action *action,
1953 struct mlx5_flow *dev_flow,
1954 const struct rte_flow_attr *attr,
1955 struct rte_flow_error *error)
1957 const struct rte_flow_action_raw_encap *encap_data;
1958 struct mlx5_flow_dv_encap_decap_resource res;
1960 encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1961 res.size = encap_data->size;
1962 memcpy(res.buf, encap_data->data, res.size);
1963 res.reformat_type = attr->egress ?
1964 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1965 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1967 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1969 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1970 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1971 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1972 return rte_flow_error_set(error, EINVAL,
1973 RTE_FLOW_ERROR_TYPE_ACTION,
1974 NULL, "can't create encap action");
1979 * Create action push VLAN.
1982 * Pointer to rte_eth_dev structure.
1983 * @param[in] vlan_tag
1984 * the vlan tag to push to the Ethernet header.
1985 * @param[in, out] dev_flow
1986 * Pointer to the mlx5_flow.
1988 * Pointer to the flow attributes.
1990 * Pointer to the error structure.
1993 * 0 on success, a negative errno value otherwise and rte_errno is set.
1996 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
1997 const struct rte_flow_attr *attr,
1998 const struct rte_vlan_hdr *vlan,
1999 struct mlx5_flow *dev_flow,
2000 struct rte_flow_error *error)
2002 struct mlx5_flow_dv_push_vlan_action_resource res;
2005 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
2008 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
2010 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
2011 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
2012 return flow_dv_push_vlan_action_resource_register
2013 (dev, &res, dev_flow, error);
2017 * Validate the modify-header actions.
2019 * @param[in] action_flags
2020 * Holds the actions detected until now.
2022 * Pointer to the modify action.
2024 * Pointer to error structure.
2027 * 0 on success, a negative errno value otherwise and rte_errno is set.
2030 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
2031 const struct rte_flow_action *action,
2032 struct rte_flow_error *error)
2034 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
2035 return rte_flow_error_set(error, EINVAL,
2036 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2037 NULL, "action configuration not set");
2038 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
2039 return rte_flow_error_set(error, EINVAL,
2040 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2041 "can't have encap action before"
2047 * Validate the modify-header MAC address actions.
2049 * @param[in] action_flags
2050 * Holds the actions detected until now.
2052 * Pointer to the modify action.
2053 * @param[in] item_flags
2054 * Holds the items detected.
2056 * Pointer to error structure.
2059 * 0 on success, a negative errno value otherwise and rte_errno is set.
2062 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
2063 const struct rte_flow_action *action,
2064 const uint64_t item_flags,
2065 struct rte_flow_error *error)
2069 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2071 if (!(item_flags & MLX5_FLOW_LAYER_L2))
2072 return rte_flow_error_set(error, EINVAL,
2073 RTE_FLOW_ERROR_TYPE_ACTION,
2075 "no L2 item in pattern");
2081 * Validate the modify-header IPv4 address actions.
2083 * @param[in] action_flags
2084 * Holds the actions detected until now.
2086 * Pointer to the modify action.
2087 * @param[in] item_flags
2088 * Holds the items detected.
2090 * Pointer to error structure.
2093 * 0 on success, a negative errno value otherwise and rte_errno is set.
2096 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
2097 const struct rte_flow_action *action,
2098 const uint64_t item_flags,
2099 struct rte_flow_error *error)
2103 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2105 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
2106 return rte_flow_error_set(error, EINVAL,
2107 RTE_FLOW_ERROR_TYPE_ACTION,
2109 "no ipv4 item in pattern");
2115 * Validate the modify-header IPv6 address actions.
2117 * @param[in] action_flags
2118 * Holds the actions detected until now.
2120 * Pointer to the modify action.
2121 * @param[in] item_flags
2122 * Holds the items detected.
2124 * Pointer to error structure.
2127 * 0 on success, a negative errno value otherwise and rte_errno is set.
2130 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
2131 const struct rte_flow_action *action,
2132 const uint64_t item_flags,
2133 struct rte_flow_error *error)
2137 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2139 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
2140 return rte_flow_error_set(error, EINVAL,
2141 RTE_FLOW_ERROR_TYPE_ACTION,
2143 "no ipv6 item in pattern");
2149 * Validate the modify-header TP actions.
2151 * @param[in] action_flags
2152 * Holds the actions detected until now.
2154 * Pointer to the modify action.
2155 * @param[in] item_flags
2156 * Holds the items detected.
2158 * Pointer to error structure.
2161 * 0 on success, a negative errno value otherwise and rte_errno is set.
2164 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
2165 const struct rte_flow_action *action,
2166 const uint64_t item_flags,
2167 struct rte_flow_error *error)
2171 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2173 if (!(item_flags & MLX5_FLOW_LAYER_L4))
2174 return rte_flow_error_set(error, EINVAL,
2175 RTE_FLOW_ERROR_TYPE_ACTION,
2176 NULL, "no transport layer "
2183 * Validate the modify-header actions of increment/decrement
2184 * TCP Sequence-number.
2186 * @param[in] action_flags
2187 * Holds the actions detected until now.
2189 * Pointer to the modify action.
2190 * @param[in] item_flags
2191 * Holds the items detected.
2193 * Pointer to error structure.
2196 * 0 on success, a negative errno value otherwise and rte_errno is set.
2199 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
2200 const struct rte_flow_action *action,
2201 const uint64_t item_flags,
2202 struct rte_flow_error *error)
2206 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2208 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2209 return rte_flow_error_set(error, EINVAL,
2210 RTE_FLOW_ERROR_TYPE_ACTION,
2211 NULL, "no TCP item in"
2213 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
2214 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
2215 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
2216 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
2217 return rte_flow_error_set(error, EINVAL,
2218 RTE_FLOW_ERROR_TYPE_ACTION,
2220 "cannot decrease and increase"
2221 " TCP sequence number"
2222 " at the same time");
2228 * Validate the modify-header actions of increment/decrement
2229 * TCP Acknowledgment number.
2231 * @param[in] action_flags
2232 * Holds the actions detected until now.
2234 * Pointer to the modify action.
2235 * @param[in] item_flags
2236 * Holds the items detected.
2238 * Pointer to error structure.
2241 * 0 on success, a negative errno value otherwise and rte_errno is set.
2244 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
2245 const struct rte_flow_action *action,
2246 const uint64_t item_flags,
2247 struct rte_flow_error *error)
2251 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2253 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2254 return rte_flow_error_set(error, EINVAL,
2255 RTE_FLOW_ERROR_TYPE_ACTION,
2256 NULL, "no TCP item in"
2258 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
2259 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
2260 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
2261 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
2262 return rte_flow_error_set(error, EINVAL,
2263 RTE_FLOW_ERROR_TYPE_ACTION,
2265 "cannot decrease and increase"
2266 " TCP acknowledgment number"
2267 " at the same time");
2273 * Validate the modify-header TTL actions.
2275 * @param[in] action_flags
2276 * Holds the actions detected until now.
2278 * Pointer to the modify action.
2279 * @param[in] item_flags
2280 * Holds the items detected.
2282 * Pointer to error structure.
2285 * 0 on success, a negative errno value otherwise and rte_errno is set.
2288 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
2289 const struct rte_flow_action *action,
2290 const uint64_t item_flags,
2291 struct rte_flow_error *error)
2295 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2297 if (!(item_flags & MLX5_FLOW_LAYER_L3))
2298 return rte_flow_error_set(error, EINVAL,
2299 RTE_FLOW_ERROR_TYPE_ACTION,
2301 "no IP protocol in pattern");
2307 * Validate jump action.
2310 * Pointer to the jump action.
2311 * @param[in] action_flags
2312 * Holds the actions detected until now.
2314 * The group of the current flow.
2316 * Pointer to error structure.
2319 * 0 on success, a negative errno value otherwise and rte_errno is set.
2322 flow_dv_validate_action_jump(const struct rte_flow_action *action,
2323 uint64_t action_flags,
2325 struct rte_flow_error *error)
2327 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2328 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2329 return rte_flow_error_set(error, EINVAL,
2330 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2331 "can't have 2 fate actions in"
2334 return rte_flow_error_set(error, EINVAL,
2335 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2336 NULL, "action configuration not set");
2337 if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
2338 return rte_flow_error_set(error, EINVAL,
2339 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2340 "target group must be higher then"
2341 " the current flow group");
2346 * Validate the port_id action.
2349 * Pointer to rte_eth_dev structure.
2350 * @param[in] action_flags
2351 * Bit-fields that holds the actions detected until now.
2353 * Port_id RTE action structure.
2355 * Attributes of flow that includes this action.
2357 * Pointer to error structure.
2360 * 0 on success, a negative errno value otherwise and rte_errno is set.
2363 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2364 uint64_t action_flags,
2365 const struct rte_flow_action *action,
2366 const struct rte_flow_attr *attr,
2367 struct rte_flow_error *error)
2369 const struct rte_flow_action_port_id *port_id;
2371 uint16_t esw_domain_id;
2372 uint16_t act_port_domain_id;
2375 if (!attr->transfer)
2376 return rte_flow_error_set(error, ENOTSUP,
2377 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2379 "port id action is valid in transfer"
2381 if (!action || !action->conf)
2382 return rte_flow_error_set(error, ENOTSUP,
2383 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2385 "port id action parameters must be"
2387 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2388 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2389 return rte_flow_error_set(error, EINVAL,
2390 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2391 "can have only one fate actions in"
2393 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2394 &esw_domain_id, NULL);
2396 return rte_flow_error_set(error, -ret,
2397 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2399 "failed to obtain E-Switch info");
2400 port_id = action->conf;
2401 port = port_id->original ? dev->data->port_id : port_id->id;
2402 ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2404 return rte_flow_error_set
2406 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2407 "failed to obtain E-Switch port id for port");
2408 if (act_port_domain_id != esw_domain_id)
2409 return rte_flow_error_set
2411 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2412 "port does not belong to"
2413 " E-Switch being configured");
2418 * Find existing modify-header resource or create and register a new one.
2420 * @param dev[in, out]
2421 * Pointer to rte_eth_dev structure.
2422 * @param[in, out] resource
2423 * Pointer to modify-header resource.
2424 * @parm[in, out] dev_flow
2425 * Pointer to the dev_flow.
2427 * pointer to error structure.
2430 * 0 on success otherwise -errno and errno is set.
2433 flow_dv_modify_hdr_resource_register
2434 (struct rte_eth_dev *dev,
2435 struct mlx5_flow_dv_modify_hdr_resource *resource,
2436 struct mlx5_flow *dev_flow,
2437 struct rte_flow_error *error)
2439 struct mlx5_priv *priv = dev->data->dev_private;
2440 struct mlx5_ibv_shared *sh = priv->sh;
2441 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2442 struct mlx5dv_dr_domain *ns;
2444 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2445 ns = sh->fdb_domain;
2446 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2451 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2452 /* Lookup a matching resource from cache. */
2453 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2454 if (resource->ft_type == cache_resource->ft_type &&
2455 resource->actions_num == cache_resource->actions_num &&
2456 resource->flags == cache_resource->flags &&
2457 !memcmp((const void *)resource->actions,
2458 (const void *)cache_resource->actions,
2459 (resource->actions_num *
2460 sizeof(resource->actions[0])))) {
2461 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2462 (void *)cache_resource,
2463 rte_atomic32_read(&cache_resource->refcnt));
2464 rte_atomic32_inc(&cache_resource->refcnt);
2465 dev_flow->dv.modify_hdr = cache_resource;
2469 /* Register new modify-header resource. */
2470 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2471 if (!cache_resource)
2472 return rte_flow_error_set(error, ENOMEM,
2473 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2474 "cannot allocate resource memory");
2475 *cache_resource = *resource;
2476 cache_resource->verbs_action =
2477 mlx5_glue->dv_create_flow_action_modify_header
2478 (sh->ctx, cache_resource->ft_type,
2479 ns, cache_resource->flags,
2480 cache_resource->actions_num *
2481 sizeof(cache_resource->actions[0]),
2482 (uint64_t *)cache_resource->actions);
2483 if (!cache_resource->verbs_action) {
2484 rte_free(cache_resource);
2485 return rte_flow_error_set(error, ENOMEM,
2486 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2487 NULL, "cannot create action");
2489 rte_atomic32_init(&cache_resource->refcnt);
2490 rte_atomic32_inc(&cache_resource->refcnt);
2491 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2492 dev_flow->dv.modify_hdr = cache_resource;
2493 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2494 (void *)cache_resource,
2495 rte_atomic32_read(&cache_resource->refcnt));
2499 #define MLX5_CNT_CONTAINER_RESIZE 64
2502 * Get or create a flow counter.
2505 * Pointer to the Ethernet device structure.
2507 * Indicate if this counter is shared with other flows.
2509 * Counter identifier.
2512 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2514 static struct mlx5_flow_counter *
2515 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2518 struct mlx5_priv *priv = dev->data->dev_private;
2519 struct mlx5_flow_counter *cnt = NULL;
2520 struct mlx5_devx_obj *dcs = NULL;
2522 if (!priv->config.devx) {
2523 rte_errno = ENOTSUP;
2527 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2528 if (cnt->shared && cnt->id == id) {
2534 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2537 cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2539 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2543 struct mlx5_flow_counter tmpl = {
2549 tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2551 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2557 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2562 * Release a flow counter.
2565 * Pointer to the Ethernet device structure.
2566 * @param[in] counter
2567 * Pointer to the counter handler.
2570 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2571 struct mlx5_flow_counter *counter)
2573 struct mlx5_priv *priv = dev->data->dev_private;
2577 if (--counter->ref_cnt == 0) {
2578 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2579 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2585 * Query a devx flow counter.
2588 * Pointer to the Ethernet device structure.
2590 * Pointer to the flow counter.
2592 * The statistics value of packets.
2594 * The statistics value of bytes.
2597 * 0 on success, otherwise a negative errno value and rte_errno is set.
2600 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2601 struct mlx5_flow_counter *cnt, uint64_t *pkts,
2604 return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2609 * Get a pool by a counter.
2612 * Pointer to the counter.
2617 static struct mlx5_flow_counter_pool *
2618 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2621 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2622 return (struct mlx5_flow_counter_pool *)cnt - 1;
2628 * Get a pool by devx counter ID.
2631 * Pointer to the counter container.
2633 * The counter devx ID.
2636 * The counter pool pointer if exists, NULL otherwise,
2638 static struct mlx5_flow_counter_pool *
2639 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2641 struct mlx5_flow_counter_pool *pool;
2643 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2644 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2645 MLX5_COUNTERS_PER_POOL;
2647 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2654 * Allocate a new memory for the counter values wrapped by all the needed
2658 * Pointer to the Ethernet device structure.
2660 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2663 * The new memory management pointer on success, otherwise NULL and rte_errno
2666 static struct mlx5_counter_stats_mem_mng *
2667 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2669 struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2670 (dev->data->dev_private))->sh;
2671 struct mlx5_devx_mkey_attr mkey_attr;
2672 struct mlx5_counter_stats_mem_mng *mem_mng;
2673 volatile struct flow_counter_stats *raw_data;
2674 int size = (sizeof(struct flow_counter_stats) *
2675 MLX5_COUNTERS_PER_POOL +
2676 sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2677 sizeof(struct mlx5_counter_stats_mem_mng);
2678 uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2685 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2686 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2687 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2688 IBV_ACCESS_LOCAL_WRITE);
2689 if (!mem_mng->umem) {
2694 mkey_attr.addr = (uintptr_t)mem;
2695 mkey_attr.size = size;
2696 mkey_attr.umem_id = mem_mng->umem->umem_id;
2697 mkey_attr.pd = sh->pdn;
2698 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2700 mlx5_glue->devx_umem_dereg(mem_mng->umem);
2705 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
2706 raw_data = (volatile struct flow_counter_stats *)mem;
2707 for (i = 0; i < raws_n; ++i) {
2708 mem_mng->raws[i].mem_mng = mem_mng;
2709 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
2711 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
2716 * Resize a counter container.
2719 * Pointer to the Ethernet device structure.
2721 * Whether the pool is for counter that was allocated by batch command.
2724 * The new container pointer on success, otherwise NULL and rte_errno is set.
2726 static struct mlx5_pools_container *
2727 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
2729 struct mlx5_priv *priv = dev->data->dev_private;
2730 struct mlx5_pools_container *cont =
2731 MLX5_CNT_CONTAINER(priv->sh, batch, 0);
2732 struct mlx5_pools_container *new_cont =
2733 MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
2734 struct mlx5_counter_stats_mem_mng *mem_mng;
2735 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
2736 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
2739 if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
2740 /* The last resize still hasn't detected by the host thread. */
2744 new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
2745 if (!new_cont->pools) {
2750 memcpy(new_cont->pools, cont->pools, cont->n *
2751 sizeof(struct mlx5_flow_counter_pool *));
2752 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
2753 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
2755 rte_free(new_cont->pools);
2758 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
2759 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
2760 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
2762 new_cont->n = resize;
2763 rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
2764 TAILQ_INIT(&new_cont->pool_list);
2765 TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
2766 new_cont->init_mem_mng = mem_mng;
2768 /* Flip the master container. */
2769 priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
2774 * Query a devx flow counter.
2777 * Pointer to the Ethernet device structure.
2779 * Pointer to the flow counter.
2781 * The statistics value of packets.
2783 * The statistics value of bytes.
2786 * 0 on success, otherwise a negative errno value and rte_errno is set.
2789 _flow_dv_query_count(struct rte_eth_dev *dev,
2790 struct mlx5_flow_counter *cnt, uint64_t *pkts,
2793 struct mlx5_priv *priv = dev->data->dev_private;
2794 struct mlx5_flow_counter_pool *pool =
2795 flow_dv_counter_pool_get(cnt);
2796 int offset = cnt - &pool->counters_raw[0];
2798 if (priv->counter_fallback)
2799 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
2801 rte_spinlock_lock(&pool->sl);
2803 * The single counters allocation may allocate smaller ID than the
2804 * current allocated in parallel to the host reading.
2805 * In this case the new counter values must be reported as 0.
2807 if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
2811 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
2812 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
2814 rte_spinlock_unlock(&pool->sl);
2819 * Create and initialize a new counter pool.
2822 * Pointer to the Ethernet device structure.
2824 * The devX counter handle.
2826 * Whether the pool is for counter that was allocated by batch command.
2829 * A new pool pointer on success, NULL otherwise and rte_errno is set.
2831 static struct mlx5_flow_counter_pool *
2832 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
2835 struct mlx5_priv *priv = dev->data->dev_private;
2836 struct mlx5_flow_counter_pool *pool;
2837 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2839 int16_t n_valid = rte_atomic16_read(&cont->n_valid);
2842 if (cont->n == n_valid) {
2843 cont = flow_dv_container_resize(dev, batch);
2847 size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
2848 sizeof(struct mlx5_flow_counter);
2849 pool = rte_calloc(__func__, 1, size, 0);
2854 pool->min_dcs = dcs;
2855 pool->raw = cont->init_mem_mng->raws + n_valid %
2856 MLX5_CNT_CONTAINER_RESIZE;
2857 pool->raw_hw = NULL;
2858 rte_spinlock_init(&pool->sl);
2860 * The generation of the new allocated counters in this pool is 0, 2 in
2861 * the pool generation makes all the counters valid for allocation.
2863 rte_atomic64_set(&pool->query_gen, 0x2);
2864 TAILQ_INIT(&pool->counters);
2865 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2866 cont->pools[n_valid] = pool;
2867 /* Pool initialization must be updated before host thread access. */
2869 rte_atomic16_add(&cont->n_valid, 1);
2874 * Prepare a new counter and/or a new counter pool.
2877 * Pointer to the Ethernet device structure.
2878 * @param[out] cnt_free
2879 * Where to put the pointer of a new counter.
2881 * Whether the pool is for counter that was allocated by batch command.
2884 * The free counter pool pointer and @p cnt_free is set on success,
2885 * NULL otherwise and rte_errno is set.
2887 static struct mlx5_flow_counter_pool *
2888 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
2889 struct mlx5_flow_counter **cnt_free,
2892 struct mlx5_priv *priv = dev->data->dev_private;
2893 struct mlx5_flow_counter_pool *pool;
2894 struct mlx5_devx_obj *dcs = NULL;
2895 struct mlx5_flow_counter *cnt;
2899 /* bulk_bitmap must be 0 for single counter allocation. */
2900 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2903 pool = flow_dv_find_pool_by_id
2904 (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
2906 pool = flow_dv_pool_create(dev, dcs, batch);
2908 mlx5_devx_cmd_destroy(dcs);
2911 } else if (dcs->id < pool->min_dcs->id) {
2912 rte_atomic64_set(&pool->a64_dcs,
2913 (int64_t)(uintptr_t)dcs);
2915 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
2916 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2921 /* bulk_bitmap is in 128 counters units. */
2922 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
2923 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
2925 rte_errno = ENODATA;
2928 pool = flow_dv_pool_create(dev, dcs, batch);
2930 mlx5_devx_cmd_destroy(dcs);
2933 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2934 cnt = &pool->counters_raw[i];
2936 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2938 *cnt_free = &pool->counters_raw[0];
2943 * Search for existed shared counter.
2946 * Pointer to the relevant counter pool container.
2948 * The shared counter ID to search.
2951 * NULL if not existed, otherwise pointer to the shared counter.
2953 static struct mlx5_flow_counter *
2954 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
2957 static struct mlx5_flow_counter *cnt;
2958 struct mlx5_flow_counter_pool *pool;
2961 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2962 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2963 cnt = &pool->counters_raw[i];
2964 if (cnt->ref_cnt && cnt->shared && cnt->id == id)
2972 * Allocate a flow counter.
2975 * Pointer to the Ethernet device structure.
2977 * Indicate if this counter is shared with other flows.
2979 * Counter identifier.
2981 * Counter flow group.
2984 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2986 static struct mlx5_flow_counter *
2987 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
2990 struct mlx5_priv *priv = dev->data->dev_private;
2991 struct mlx5_flow_counter_pool *pool = NULL;
2992 struct mlx5_flow_counter *cnt_free = NULL;
2994 * Currently group 0 flow counter cannot be assigned to a flow if it is
2995 * not the first one in the batch counter allocation, so it is better
2996 * to allocate counters one by one for these flows in a separate
2998 * A counter can be shared between different groups so need to take
2999 * shared counters from the single container.
3001 uint32_t batch = (group && !shared) ? 1 : 0;
3002 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
3005 if (priv->counter_fallback)
3006 return flow_dv_counter_alloc_fallback(dev, shared, id);
3007 if (!priv->config.devx) {
3008 rte_errno = ENOTSUP;
3012 cnt_free = flow_dv_counter_shared_search(cont, id);
3014 if (cnt_free->ref_cnt + 1 == 0) {
3018 cnt_free->ref_cnt++;
3022 /* Pools which has a free counters are in the start. */
3023 TAILQ_FOREACH(pool, &cont->pool_list, next) {
3025 * The free counter reset values must be updated between the
3026 * counter release to the counter allocation, so, at least one
3027 * query must be done in this time. ensure it by saving the
3028 * query generation in the release time.
3029 * The free list is sorted according to the generation - so if
3030 * the first one is not updated, all the others are not
3033 cnt_free = TAILQ_FIRST(&pool->counters);
3034 if (cnt_free && cnt_free->query_gen + 1 <
3035 rte_atomic64_read(&pool->query_gen))
3040 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
3044 cnt_free->batch = batch;
3045 /* Create a DV counter action only in the first time usage. */
3046 if (!cnt_free->action) {
3048 struct mlx5_devx_obj *dcs;
3051 offset = cnt_free - &pool->counters_raw[0];
3052 dcs = pool->min_dcs;
3055 dcs = cnt_free->dcs;
3057 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
3059 if (!cnt_free->action) {
3064 /* Update the counter reset values. */
3065 if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
3068 cnt_free->shared = shared;
3069 cnt_free->ref_cnt = 1;
3071 if (!priv->sh->cmng.query_thread_on)
3072 /* Start the asynchronous batch query by the host thread. */
3073 mlx5_set_query_alarm(priv->sh);
3074 TAILQ_REMOVE(&pool->counters, cnt_free, next);
3075 if (TAILQ_EMPTY(&pool->counters)) {
3076 /* Move the pool to the end of the container pool list. */
3077 TAILQ_REMOVE(&cont->pool_list, pool, next);
3078 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
3084 * Release a flow counter.
3087 * Pointer to the Ethernet device structure.
3088 * @param[in] counter
3089 * Pointer to the counter handler.
3092 flow_dv_counter_release(struct rte_eth_dev *dev,
3093 struct mlx5_flow_counter *counter)
3095 struct mlx5_priv *priv = dev->data->dev_private;
3099 if (priv->counter_fallback) {
3100 flow_dv_counter_release_fallback(dev, counter);
3103 if (--counter->ref_cnt == 0) {
3104 struct mlx5_flow_counter_pool *pool =
3105 flow_dv_counter_pool_get(counter);
3107 /* Put the counter in the end - the last updated one. */
3108 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
3109 counter->query_gen = rte_atomic64_read(&pool->query_gen);
3114 * Verify the @p attributes will be correctly understood by the NIC and store
3115 * them in the @p flow if everything is correct.
3118 * Pointer to dev struct.
3119 * @param[in] attributes
3120 * Pointer to flow attributes
3122 * Pointer to error structure.
3125 * 0 on success, a negative errno value otherwise and rte_errno is set.
3128 flow_dv_validate_attributes(struct rte_eth_dev *dev,
3129 const struct rte_flow_attr *attributes,
3130 struct rte_flow_error *error)
3132 struct mlx5_priv *priv = dev->data->dev_private;
3133 uint32_t priority_max = priv->config.flow_prio - 1;
3135 #ifndef HAVE_MLX5DV_DR
3136 if (attributes->group)
3137 return rte_flow_error_set(error, ENOTSUP,
3138 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
3140 "groups is not supported");
3142 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
3143 attributes->priority >= priority_max)
3144 return rte_flow_error_set(error, ENOTSUP,
3145 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
3147 "priority out of range");
3148 if (attributes->transfer) {
3149 if (!priv->config.dv_esw_en)
3150 return rte_flow_error_set
3152 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3153 "E-Switch dr is not supported");
3154 if (!(priv->representor || priv->master))
3155 return rte_flow_error_set
3156 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3157 NULL, "E-Switch configuration can only be"
3158 " done by a master or a representor device");
3159 if (attributes->egress)
3160 return rte_flow_error_set
3162 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
3163 "egress is not supported");
3164 if (attributes->group >= MLX5_MAX_TABLES_FDB)
3165 return rte_flow_error_set
3167 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
3168 NULL, "group must be smaller than "
3169 RTE_STR(MLX5_MAX_TABLES_FDB));
3171 if (!(attributes->egress ^ attributes->ingress))
3172 return rte_flow_error_set(error, ENOTSUP,
3173 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
3174 "must specify exactly one of "
3175 "ingress or egress");
3180 * Internal validation function. For validating both actions and items.
3183 * Pointer to the rte_eth_dev structure.
3185 * Pointer to the flow attributes.
3187 * Pointer to the list of items.
3188 * @param[in] actions
3189 * Pointer to the list of actions.
3191 * Pointer to the error structure.
3194 * 0 on success, a negative errno value otherwise and rte_errno is set.
3197 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
3198 const struct rte_flow_item items[],
3199 const struct rte_flow_action actions[],
3200 struct rte_flow_error *error)
3203 uint64_t action_flags = 0;
3204 uint64_t item_flags = 0;
3205 uint64_t last_item = 0;
3206 uint8_t next_protocol = 0xff;
3208 const struct rte_flow_item *gre_item = NULL;
3209 struct rte_flow_item_tcp nic_tcp_mask = {
3212 .src_port = RTE_BE16(UINT16_MAX),
3213 .dst_port = RTE_BE16(UINT16_MAX),
3219 ret = flow_dv_validate_attributes(dev, attr, error);
3222 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3223 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3224 switch (items->type) {
3225 case RTE_FLOW_ITEM_TYPE_VOID:
3227 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3228 ret = flow_dv_validate_item_port_id
3229 (dev, items, attr, item_flags, error);
3232 last_item = MLX5_FLOW_ITEM_PORT_ID;
3234 case RTE_FLOW_ITEM_TYPE_ETH:
3235 ret = mlx5_flow_validate_item_eth(items, item_flags,
3239 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
3240 MLX5_FLOW_LAYER_OUTER_L2;
3242 case RTE_FLOW_ITEM_TYPE_VLAN:
3243 ret = mlx5_flow_validate_item_vlan(items, item_flags,
3247 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
3248 MLX5_FLOW_LAYER_OUTER_VLAN;
3250 case RTE_FLOW_ITEM_TYPE_IPV4:
3251 mlx5_flow_tunnel_ip_check(items, next_protocol,
3252 &item_flags, &tunnel);
3253 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
3257 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3258 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3259 if (items->mask != NULL &&
3260 ((const struct rte_flow_item_ipv4 *)
3261 items->mask)->hdr.next_proto_id) {
3263 ((const struct rte_flow_item_ipv4 *)
3264 (items->spec))->hdr.next_proto_id;
3266 ((const struct rte_flow_item_ipv4 *)
3267 (items->mask))->hdr.next_proto_id;
3269 /* Reset for inner layer. */
3270 next_protocol = 0xff;
3273 case RTE_FLOW_ITEM_TYPE_IPV6:
3274 mlx5_flow_tunnel_ip_check(items, next_protocol,
3275 &item_flags, &tunnel);
3276 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
3280 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3281 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3282 if (items->mask != NULL &&
3283 ((const struct rte_flow_item_ipv6 *)
3284 items->mask)->hdr.proto) {
3286 ((const struct rte_flow_item_ipv6 *)
3287 items->spec)->hdr.proto;
3289 ((const struct rte_flow_item_ipv6 *)
3290 items->mask)->hdr.proto;
3292 /* Reset for inner layer. */
3293 next_protocol = 0xff;
3296 case RTE_FLOW_ITEM_TYPE_TCP:
3297 ret = mlx5_flow_validate_item_tcp
3304 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3305 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3307 case RTE_FLOW_ITEM_TYPE_UDP:
3308 ret = mlx5_flow_validate_item_udp(items, item_flags,
3313 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3314 MLX5_FLOW_LAYER_OUTER_L4_UDP;
3316 case RTE_FLOW_ITEM_TYPE_GRE:
3317 ret = mlx5_flow_validate_item_gre(items, item_flags,
3318 next_protocol, error);
3322 last_item = MLX5_FLOW_LAYER_GRE;
3324 case RTE_FLOW_ITEM_TYPE_NVGRE:
3325 ret = mlx5_flow_validate_item_nvgre(items, item_flags,
3330 last_item = MLX5_FLOW_LAYER_NVGRE;
3332 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
3333 ret = mlx5_flow_validate_item_gre_key
3334 (items, item_flags, gre_item, error);
3337 last_item = MLX5_FLOW_LAYER_GRE_KEY;
3339 case RTE_FLOW_ITEM_TYPE_VXLAN:
3340 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
3344 last_item = MLX5_FLOW_LAYER_VXLAN;
3346 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3347 ret = mlx5_flow_validate_item_vxlan_gpe(items,
3352 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3354 case RTE_FLOW_ITEM_TYPE_MPLS:
3355 ret = mlx5_flow_validate_item_mpls(dev, items,
3360 last_item = MLX5_FLOW_LAYER_MPLS;
3362 case RTE_FLOW_ITEM_TYPE_META:
3363 ret = flow_dv_validate_item_meta(dev, items, attr,
3367 last_item = MLX5_FLOW_ITEM_METADATA;
3369 case RTE_FLOW_ITEM_TYPE_ICMP:
3370 ret = mlx5_flow_validate_item_icmp(items, item_flags,
3375 last_item = MLX5_FLOW_LAYER_ICMP;
3377 case RTE_FLOW_ITEM_TYPE_ICMP6:
3378 ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3383 last_item = MLX5_FLOW_LAYER_ICMP6;
3386 return rte_flow_error_set(error, ENOTSUP,
3387 RTE_FLOW_ERROR_TYPE_ITEM,
3388 NULL, "item not supported");
3390 item_flags |= last_item;
3392 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3393 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3394 return rte_flow_error_set(error, ENOTSUP,
3395 RTE_FLOW_ERROR_TYPE_ACTION,
3396 actions, "too many actions");
3397 switch (actions->type) {
3398 case RTE_FLOW_ACTION_TYPE_VOID:
3400 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3401 ret = flow_dv_validate_action_port_id(dev,
3408 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3411 case RTE_FLOW_ACTION_TYPE_FLAG:
3412 ret = mlx5_flow_validate_action_flag(action_flags,
3416 action_flags |= MLX5_FLOW_ACTION_FLAG;
3419 case RTE_FLOW_ACTION_TYPE_MARK:
3420 ret = mlx5_flow_validate_action_mark(actions,
3425 action_flags |= MLX5_FLOW_ACTION_MARK;
3428 case RTE_FLOW_ACTION_TYPE_DROP:
3429 ret = mlx5_flow_validate_action_drop(action_flags,
3433 action_flags |= MLX5_FLOW_ACTION_DROP;
3436 case RTE_FLOW_ACTION_TYPE_QUEUE:
3437 ret = mlx5_flow_validate_action_queue(actions,
3442 action_flags |= MLX5_FLOW_ACTION_QUEUE;
3445 case RTE_FLOW_ACTION_TYPE_RSS:
3446 ret = mlx5_flow_validate_action_rss(actions,
3452 action_flags |= MLX5_FLOW_ACTION_RSS;
3455 case RTE_FLOW_ACTION_TYPE_COUNT:
3456 ret = flow_dv_validate_action_count(dev, error);
3459 action_flags |= MLX5_FLOW_ACTION_COUNT;
3462 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3463 if (flow_dv_validate_action_pop_vlan(dev,
3469 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
3472 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3473 ret = flow_dv_validate_action_push_vlan(action_flags,
3478 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
3481 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3482 ret = flow_dv_validate_action_set_vlan_pcp
3483 (action_flags, actions, error);
3486 /* Count PCP with push_vlan command. */
3488 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3489 ret = flow_dv_validate_action_set_vlan_vid
3490 (action_flags, actions, error);
3493 /* Count VID with push_vlan command. */
3495 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3496 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3497 ret = flow_dv_validate_action_l2_encap(action_flags,
3502 action_flags |= actions->type ==
3503 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3504 MLX5_FLOW_ACTION_VXLAN_ENCAP :
3505 MLX5_FLOW_ACTION_NVGRE_ENCAP;
3508 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3509 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3510 ret = flow_dv_validate_action_l2_decap(action_flags,
3514 action_flags |= actions->type ==
3515 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3516 MLX5_FLOW_ACTION_VXLAN_DECAP :
3517 MLX5_FLOW_ACTION_NVGRE_DECAP;
3520 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3521 ret = flow_dv_validate_action_raw_encap(action_flags,
3526 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3529 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3530 ret = flow_dv_validate_action_raw_decap(action_flags,
3535 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3538 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3539 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3540 ret = flow_dv_validate_action_modify_mac(action_flags,
3546 /* Count all modify-header actions as one action. */
3547 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3549 action_flags |= actions->type ==
3550 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3551 MLX5_FLOW_ACTION_SET_MAC_SRC :
3552 MLX5_FLOW_ACTION_SET_MAC_DST;
3555 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3556 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3557 ret = flow_dv_validate_action_modify_ipv4(action_flags,
3563 /* Count all modify-header actions as one action. */
3564 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3566 action_flags |= actions->type ==
3567 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3568 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3569 MLX5_FLOW_ACTION_SET_IPV4_DST;
3571 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3572 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3573 ret = flow_dv_validate_action_modify_ipv6(action_flags,
3579 /* Count all modify-header actions as one action. */
3580 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3582 action_flags |= actions->type ==
3583 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3584 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3585 MLX5_FLOW_ACTION_SET_IPV6_DST;
3587 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3588 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3589 ret = flow_dv_validate_action_modify_tp(action_flags,
3595 /* Count all modify-header actions as one action. */
3596 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3598 action_flags |= actions->type ==
3599 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3600 MLX5_FLOW_ACTION_SET_TP_SRC :
3601 MLX5_FLOW_ACTION_SET_TP_DST;
3603 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3604 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3605 ret = flow_dv_validate_action_modify_ttl(action_flags,
3611 /* Count all modify-header actions as one action. */
3612 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3614 action_flags |= actions->type ==
3615 RTE_FLOW_ACTION_TYPE_SET_TTL ?
3616 MLX5_FLOW_ACTION_SET_TTL :
3617 MLX5_FLOW_ACTION_DEC_TTL;
3619 case RTE_FLOW_ACTION_TYPE_JUMP:
3620 ret = flow_dv_validate_action_jump(actions,
3622 attr->group, error);
3626 action_flags |= MLX5_FLOW_ACTION_JUMP;
3628 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3629 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3630 ret = flow_dv_validate_action_modify_tcp_seq
3637 /* Count all modify-header actions as one action. */
3638 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3640 action_flags |= actions->type ==
3641 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
3642 MLX5_FLOW_ACTION_INC_TCP_SEQ :
3643 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
3645 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3646 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3647 ret = flow_dv_validate_action_modify_tcp_ack
3654 /* Count all modify-header actions as one action. */
3655 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3657 action_flags |= actions->type ==
3658 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
3659 MLX5_FLOW_ACTION_INC_TCP_ACK :
3660 MLX5_FLOW_ACTION_DEC_TCP_ACK;
3663 return rte_flow_error_set(error, ENOTSUP,
3664 RTE_FLOW_ERROR_TYPE_ACTION,
3666 "action not supported");
3669 if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3670 (action_flags & MLX5_FLOW_VLAN_ACTIONS))
3671 return rte_flow_error_set(error, ENOTSUP,
3672 RTE_FLOW_ERROR_TYPE_ACTION,
3674 "can't have vxlan and vlan"
3675 " actions in the same rule");
3676 /* Eswitch has few restrictions on using items and actions */
3677 if (attr->transfer) {
3678 if (action_flags & MLX5_FLOW_ACTION_FLAG)
3679 return rte_flow_error_set(error, ENOTSUP,
3680 RTE_FLOW_ERROR_TYPE_ACTION,
3682 "unsupported action FLAG");
3683 if (action_flags & MLX5_FLOW_ACTION_MARK)
3684 return rte_flow_error_set(error, ENOTSUP,
3685 RTE_FLOW_ERROR_TYPE_ACTION,
3687 "unsupported action MARK");
3688 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
3689 return rte_flow_error_set(error, ENOTSUP,
3690 RTE_FLOW_ERROR_TYPE_ACTION,
3692 "unsupported action QUEUE");
3693 if (action_flags & MLX5_FLOW_ACTION_RSS)
3694 return rte_flow_error_set(error, ENOTSUP,
3695 RTE_FLOW_ERROR_TYPE_ACTION,
3697 "unsupported action RSS");
3698 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3699 return rte_flow_error_set(error, EINVAL,
3700 RTE_FLOW_ERROR_TYPE_ACTION,
3702 "no fate action is found");
3704 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
3705 return rte_flow_error_set(error, EINVAL,
3706 RTE_FLOW_ERROR_TYPE_ACTION,
3708 "no fate action is found");
3714 * Internal preparation function. Allocates the DV flow size,
3715 * this size is constant.
3718 * Pointer to the flow attributes.
3720 * Pointer to the list of items.
3721 * @param[in] actions
3722 * Pointer to the list of actions.
3724 * Pointer to the error structure.
3727 * Pointer to mlx5_flow object on success,
3728 * otherwise NULL and rte_errno is set.
3730 static struct mlx5_flow *
3731 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
3732 const struct rte_flow_item items[] __rte_unused,
3733 const struct rte_flow_action actions[] __rte_unused,
3734 struct rte_flow_error *error)
3736 uint32_t size = sizeof(struct mlx5_flow);
3737 struct mlx5_flow *flow;
3739 flow = rte_calloc(__func__, 1, size, 0);
3741 rte_flow_error_set(error, ENOMEM,
3742 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3743 "not enough memory to create flow");
3746 flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
3752 * Sanity check for match mask and value. Similar to check_valid_spec() in
3753 * kernel driver. If unmasked bit is present in value, it returns failure.
3756 * pointer to match mask buffer.
3757 * @param match_value
3758 * pointer to match value buffer.
3761 * 0 if valid, -EINVAL otherwise.
3764 flow_dv_check_valid_spec(void *match_mask, void *match_value)
3766 uint8_t *m = match_mask;
3767 uint8_t *v = match_value;
3770 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
3773 "match_value differs from match_criteria"
3774 " %p[%u] != %p[%u]",
3775 match_value, i, match_mask, i);
3784 * Add Ethernet item to matcher and to the value.
3786 * @param[in, out] matcher
3788 * @param[in, out] key
3789 * Flow matcher value.
3791 * Flow pattern to translate.
3793 * Item is inner pattern.
3796 flow_dv_translate_item_eth(void *matcher, void *key,
3797 const struct rte_flow_item *item, int inner)
3799 const struct rte_flow_item_eth *eth_m = item->mask;
3800 const struct rte_flow_item_eth *eth_v = item->spec;
3801 const struct rte_flow_item_eth nic_mask = {
3802 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3803 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3804 .type = RTE_BE16(0xffff),
3816 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3818 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3820 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3822 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3824 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
3825 ð_m->dst, sizeof(eth_m->dst));
3826 /* The value must be in the range of the mask. */
3827 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
3828 for (i = 0; i < sizeof(eth_m->dst); ++i)
3829 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
3830 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
3831 ð_m->src, sizeof(eth_m->src));
3832 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
3833 /* The value must be in the range of the mask. */
3834 for (i = 0; i < sizeof(eth_m->dst); ++i)
3835 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
3836 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3837 rte_be_to_cpu_16(eth_m->type));
3838 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
3839 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
3843 * Add VLAN item to matcher and to the value.
3845 * @param[in, out] dev_flow
3847 * @param[in, out] matcher
3849 * @param[in, out] key
3850 * Flow matcher value.
3852 * Flow pattern to translate.
3854 * Item is inner pattern.
3857 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
3858 void *matcher, void *key,
3859 const struct rte_flow_item *item,
3862 const struct rte_flow_item_vlan *vlan_m = item->mask;
3863 const struct rte_flow_item_vlan *vlan_v = item->spec;
3872 vlan_m = &rte_flow_item_vlan_mask;
3874 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3876 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3878 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3880 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3882 * This is workaround, masks are not supported,
3883 * and pre-validated.
3885 dev_flow->dv.vf_vlan.tag =
3886 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
3888 tci_m = rte_be_to_cpu_16(vlan_m->tci);
3889 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
3890 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
3891 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
3892 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
3893 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
3894 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
3895 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
3896 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
3897 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
3898 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3899 rte_be_to_cpu_16(vlan_m->inner_type));
3900 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
3901 rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
3905 * Add IPV4 item to matcher and to the value.
3907 * @param[in, out] matcher
3909 * @param[in, out] key
3910 * Flow matcher value.
3912 * Flow pattern to translate.
3914 * Item is inner pattern.
3916 * The group to insert the rule.
3919 flow_dv_translate_item_ipv4(void *matcher, void *key,
3920 const struct rte_flow_item *item,
3921 int inner, uint32_t group)
3923 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
3924 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
3925 const struct rte_flow_item_ipv4 nic_mask = {
3927 .src_addr = RTE_BE32(0xffffffff),
3928 .dst_addr = RTE_BE32(0xffffffff),
3929 .type_of_service = 0xff,
3930 .next_proto_id = 0xff,
3940 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3942 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3944 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3946 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3949 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3951 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
3952 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
3957 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3958 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3959 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3960 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3961 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
3962 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
3963 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3964 src_ipv4_src_ipv6.ipv4_layout.ipv4);
3965 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3966 src_ipv4_src_ipv6.ipv4_layout.ipv4);
3967 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
3968 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
3969 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
3970 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
3971 ipv4_m->hdr.type_of_service);
3972 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
3973 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
3974 ipv4_m->hdr.type_of_service >> 2);
3975 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
3976 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3977 ipv4_m->hdr.next_proto_id);
3978 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3979 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3983 * Add IPV6 item to matcher and to the value.
3985 * @param[in, out] matcher
3987 * @param[in, out] key
3988 * Flow matcher value.
3990 * Flow pattern to translate.
3992 * Item is inner pattern.
3994 * The group to insert the rule.
3997 flow_dv_translate_item_ipv6(void *matcher, void *key,
3998 const struct rte_flow_item *item,
3999 int inner, uint32_t group)
4001 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
4002 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
4003 const struct rte_flow_item_ipv6 nic_mask = {
4006 "\xff\xff\xff\xff\xff\xff\xff\xff"
4007 "\xff\xff\xff\xff\xff\xff\xff\xff",
4009 "\xff\xff\xff\xff\xff\xff\xff\xff"
4010 "\xff\xff\xff\xff\xff\xff\xff\xff",
4011 .vtc_flow = RTE_BE32(0xffffffff),
4018 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4019 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4028 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4030 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4032 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4034 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4037 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
4039 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
4040 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
4045 size = sizeof(ipv6_m->hdr.dst_addr);
4046 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4047 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
4048 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4049 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
4050 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
4051 for (i = 0; i < size; ++i)
4052 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
4053 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
4054 src_ipv4_src_ipv6.ipv6_layout.ipv6);
4055 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
4056 src_ipv4_src_ipv6.ipv6_layout.ipv6);
4057 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
4058 for (i = 0; i < size; ++i)
4059 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
4061 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
4062 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
4063 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
4064 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
4065 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
4066 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
4069 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
4071 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
4074 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
4076 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
4080 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
4082 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4083 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
4087 * Add TCP item to matcher and to the value.
4089 * @param[in, out] matcher
4091 * @param[in, out] key
4092 * Flow matcher value.
4094 * Flow pattern to translate.
4096 * Item is inner pattern.
4099 flow_dv_translate_item_tcp(void *matcher, void *key,
4100 const struct rte_flow_item *item,
4103 const struct rte_flow_item_tcp *tcp_m = item->mask;
4104 const struct rte_flow_item_tcp *tcp_v = item->spec;
4109 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4111 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4113 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4115 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4117 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4118 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
4122 tcp_m = &rte_flow_item_tcp_mask;
4123 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
4124 rte_be_to_cpu_16(tcp_m->hdr.src_port));
4125 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
4126 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
4127 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
4128 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
4129 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
4130 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
4131 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
4132 tcp_m->hdr.tcp_flags);
4133 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
4134 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
4138 * Add UDP item to matcher and to the value.
4140 * @param[in, out] matcher
4142 * @param[in, out] key
4143 * Flow matcher value.
4145 * Flow pattern to translate.
4147 * Item is inner pattern.
4150 flow_dv_translate_item_udp(void *matcher, void *key,
4151 const struct rte_flow_item *item,
4154 const struct rte_flow_item_udp *udp_m = item->mask;
4155 const struct rte_flow_item_udp *udp_v = item->spec;
4160 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4162 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4164 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4166 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4168 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4169 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
4173 udp_m = &rte_flow_item_udp_mask;
4174 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
4175 rte_be_to_cpu_16(udp_m->hdr.src_port));
4176 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
4177 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
4178 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
4179 rte_be_to_cpu_16(udp_m->hdr.dst_port));
4180 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4181 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
4185 * Add GRE optional Key item to matcher and to the value.
4187 * @param[in, out] matcher
4189 * @param[in, out] key
4190 * Flow matcher value.
4192 * Flow pattern to translate.
4194 * Item is inner pattern.
4197 flow_dv_translate_item_gre_key(void *matcher, void *key,
4198 const struct rte_flow_item *item)
4200 const rte_be32_t *key_m = item->mask;
4201 const rte_be32_t *key_v = item->spec;
4202 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4203 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4204 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
4209 key_m = &gre_key_default_mask;
4210 /* GRE K bit must be on and should already be validated */
4211 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
4212 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
4213 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
4214 rte_be_to_cpu_32(*key_m) >> 8);
4215 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
4216 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
4217 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
4218 rte_be_to_cpu_32(*key_m) & 0xFF);
4219 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
4220 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
4224 * Add GRE item to matcher and to the value.
4226 * @param[in, out] matcher
4228 * @param[in, out] key
4229 * Flow matcher value.
4231 * Flow pattern to translate.
4233 * Item is inner pattern.
4236 flow_dv_translate_item_gre(void *matcher, void *key,
4237 const struct rte_flow_item *item,
4240 const struct rte_flow_item_gre *gre_m = item->mask;
4241 const struct rte_flow_item_gre *gre_v = item->spec;
4244 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4245 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4252 uint16_t s_present:1;
4253 uint16_t k_present:1;
4254 uint16_t rsvd_bit1:1;
4255 uint16_t c_present:1;
4259 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
4262 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4264 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4266 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4268 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4270 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4271 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
4275 gre_m = &rte_flow_item_gre_mask;
4276 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
4277 rte_be_to_cpu_16(gre_m->protocol));
4278 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4279 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
4280 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
4281 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
4282 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
4283 gre_crks_rsvd0_ver_m.c_present);
4284 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
4285 gre_crks_rsvd0_ver_v.c_present &
4286 gre_crks_rsvd0_ver_m.c_present);
4287 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
4288 gre_crks_rsvd0_ver_m.k_present);
4289 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
4290 gre_crks_rsvd0_ver_v.k_present &
4291 gre_crks_rsvd0_ver_m.k_present);
4292 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
4293 gre_crks_rsvd0_ver_m.s_present);
4294 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
4295 gre_crks_rsvd0_ver_v.s_present &
4296 gre_crks_rsvd0_ver_m.s_present);
4300 * Add NVGRE item to matcher and to the value.
4302 * @param[in, out] matcher
4304 * @param[in, out] key
4305 * Flow matcher value.
4307 * Flow pattern to translate.
4309 * Item is inner pattern.
4312 flow_dv_translate_item_nvgre(void *matcher, void *key,
4313 const struct rte_flow_item *item,
4316 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
4317 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
4318 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4319 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4320 const char *tni_flow_id_m = (const char *)nvgre_m->tni;
4321 const char *tni_flow_id_v = (const char *)nvgre_v->tni;
4327 /* For NVGRE, GRE header fields must be set with defined values. */
4328 const struct rte_flow_item_gre gre_spec = {
4329 .c_rsvd0_ver = RTE_BE16(0x2000),
4330 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
4332 const struct rte_flow_item_gre gre_mask = {
4333 .c_rsvd0_ver = RTE_BE16(0xB000),
4334 .protocol = RTE_BE16(UINT16_MAX),
4336 const struct rte_flow_item gre_item = {
4341 flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
4345 nvgre_m = &rte_flow_item_nvgre_mask;
4346 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
4347 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
4348 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
4349 memcpy(gre_key_m, tni_flow_id_m, size);
4350 for (i = 0; i < size; ++i)
4351 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
4355 * Add VXLAN item to matcher and to the value.
4357 * @param[in, out] matcher
4359 * @param[in, out] key
4360 * Flow matcher value.
4362 * Flow pattern to translate.
4364 * Item is inner pattern.
4367 flow_dv_translate_item_vxlan(void *matcher, void *key,
4368 const struct rte_flow_item *item,
4371 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
4372 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
4375 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4376 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4384 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4386 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4388 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4390 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4392 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
4393 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
4394 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
4395 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
4396 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
4401 vxlan_m = &rte_flow_item_vxlan_mask;
4402 size = sizeof(vxlan_m->vni);
4403 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4404 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4405 memcpy(vni_m, vxlan_m->vni, size);
4406 for (i = 0; i < size; ++i)
4407 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4411 * Add MPLS item to matcher and to the value.
4413 * @param[in, out] matcher
4415 * @param[in, out] key
4416 * Flow matcher value.
4418 * Flow pattern to translate.
4419 * @param[in] prev_layer
4420 * The protocol layer indicated in previous item.
4422 * Item is inner pattern.
4425 flow_dv_translate_item_mpls(void *matcher, void *key,
4426 const struct rte_flow_item *item,
4427 uint64_t prev_layer,
4430 const uint32_t *in_mpls_m = item->mask;
4431 const uint32_t *in_mpls_v = item->spec;
4432 uint32_t *out_mpls_m = 0;
4433 uint32_t *out_mpls_v = 0;
4434 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4435 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4436 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4438 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4439 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4440 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4442 switch (prev_layer) {
4443 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4444 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4445 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4446 MLX5_UDP_PORT_MPLS);
4448 case MLX5_FLOW_LAYER_GRE:
4449 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4450 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4451 RTE_ETHER_TYPE_MPLS);
4454 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4455 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4462 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4463 switch (prev_layer) {
4464 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4466 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4467 outer_first_mpls_over_udp);
4469 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4470 outer_first_mpls_over_udp);
4472 case MLX5_FLOW_LAYER_GRE:
4474 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4475 outer_first_mpls_over_gre);
4477 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4478 outer_first_mpls_over_gre);
4481 /* Inner MPLS not over GRE is not supported. */
4484 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4488 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4494 if (out_mpls_m && out_mpls_v) {
4495 *out_mpls_m = *in_mpls_m;
4496 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4501 * Add META item to matcher
4503 * @param[in, out] matcher
4505 * @param[in, out] key
4506 * Flow matcher value.
4508 * Flow pattern to translate.
4510 * Item is inner pattern.
4513 flow_dv_translate_item_meta(void *matcher, void *key,
4514 const struct rte_flow_item *item)
4516 const struct rte_flow_item_meta *meta_m;
4517 const struct rte_flow_item_meta *meta_v;
4519 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4521 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4523 meta_m = (const void *)item->mask;
4525 meta_m = &rte_flow_item_meta_mask;
4526 meta_v = (const void *)item->spec;
4528 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
4529 rte_be_to_cpu_32(meta_m->data));
4530 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
4531 rte_be_to_cpu_32(meta_v->data & meta_m->data));
4536 * Add source vport match to the specified matcher.
4538 * @param[in, out] matcher
4540 * @param[in, out] key
4541 * Flow matcher value.
4543 * Source vport value to match
4548 flow_dv_translate_item_source_vport(void *matcher, void *key,
4549 int16_t port, uint16_t mask)
4551 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4552 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4554 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
4555 MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
4559 * Translate port-id item to eswitch match on port-id.
4562 * The devich to configure through.
4563 * @param[in, out] matcher
4565 * @param[in, out] key
4566 * Flow matcher value.
4568 * Flow pattern to translate.
4571 * 0 on success, a negative errno value otherwise.
4574 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
4575 void *key, const struct rte_flow_item *item)
4577 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
4578 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
4579 uint16_t mask, val, id;
4582 mask = pid_m ? pid_m->id : 0xffff;
4583 id = pid_v ? pid_v->id : dev->data->port_id;
4584 ret = mlx5_port_to_eswitch_info(id, NULL, &val);
4587 flow_dv_translate_item_source_vport(matcher, key, val, mask);
4592 * Add ICMP6 item to matcher and to the value.
4594 * @param[in, out] matcher
4596 * @param[in, out] key
4597 * Flow matcher value.
4599 * Flow pattern to translate.
4601 * Item is inner pattern.
4604 flow_dv_translate_item_icmp6(void *matcher, void *key,
4605 const struct rte_flow_item *item,
4608 const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
4609 const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
4612 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4614 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4616 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4618 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4620 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4622 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4624 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4625 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
4629 icmp6_m = &rte_flow_item_icmp6_mask;
4630 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
4631 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
4632 icmp6_v->type & icmp6_m->type);
4633 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
4634 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
4635 icmp6_v->code & icmp6_m->code);
4639 * Add ICMP item to matcher and to the value.
4641 * @param[in, out] matcher
4643 * @param[in, out] key
4644 * Flow matcher value.
4646 * Flow pattern to translate.
4648 * Item is inner pattern.
4651 flow_dv_translate_item_icmp(void *matcher, void *key,
4652 const struct rte_flow_item *item,
4655 const struct rte_flow_item_icmp *icmp_m = item->mask;
4656 const struct rte_flow_item_icmp *icmp_v = item->spec;
4659 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4661 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4663 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4665 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4667 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4669 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4671 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4672 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
4676 icmp_m = &rte_flow_item_icmp_mask;
4677 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
4678 icmp_m->hdr.icmp_type);
4679 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
4680 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
4681 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
4682 icmp_m->hdr.icmp_code);
4683 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
4684 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
4687 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
4689 #define HEADER_IS_ZERO(match_criteria, headers) \
4690 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
4691 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
4694 * Calculate flow matcher enable bitmap.
4696 * @param match_criteria
4697 * Pointer to flow matcher criteria.
4700 * Bitmap of enabled fields.
4703 flow_dv_matcher_enable(uint32_t *match_criteria)
4705 uint8_t match_criteria_enable;
4707 match_criteria_enable =
4708 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
4709 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
4710 match_criteria_enable |=
4711 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
4712 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
4713 match_criteria_enable |=
4714 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
4715 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
4716 match_criteria_enable |=
4717 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
4718 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
4719 match_criteria_enable |=
4720 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
4721 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
4722 return match_criteria_enable;
4729 * @param dev[in, out]
4730 * Pointer to rte_eth_dev structure.
4731 * @param[in] table_id
4734 * Direction of the table.
4735 * @param[in] transfer
4736 * E-Switch or NIC flow.
4738 * pointer to error structure.
4741 * Returns tables resource based on the index, NULL in case of failed.
4743 static struct mlx5_flow_tbl_resource *
4744 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
4745 uint32_t table_id, uint8_t egress,
4747 struct rte_flow_error *error)
4749 struct mlx5_priv *priv = dev->data->dev_private;
4750 struct mlx5_ibv_shared *sh = priv->sh;
4751 struct mlx5_flow_tbl_resource *tbl;
4753 #ifdef HAVE_MLX5DV_DR
4755 tbl = &sh->fdb_tbl[table_id];
4757 tbl->obj = mlx5_glue->dr_create_flow_tbl
4758 (sh->fdb_domain, table_id);
4759 } else if (egress) {
4760 tbl = &sh->tx_tbl[table_id];
4762 tbl->obj = mlx5_glue->dr_create_flow_tbl
4763 (sh->tx_domain, table_id);
4765 tbl = &sh->rx_tbl[table_id];
4767 tbl->obj = mlx5_glue->dr_create_flow_tbl
4768 (sh->rx_domain, table_id);
4771 rte_flow_error_set(error, ENOMEM,
4772 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4773 NULL, "cannot create table");
4776 rte_atomic32_inc(&tbl->refcnt);
4782 return &sh->fdb_tbl[table_id];
4784 return &sh->tx_tbl[table_id];
4786 return &sh->rx_tbl[table_id];
4791 * Release a flow table.
4794 * Table resource to be released.
4797 * Returns 0 if table was released, else return 1;
4800 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
4804 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
4805 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
4813 * Register the flow matcher.
4815 * @param dev[in, out]
4816 * Pointer to rte_eth_dev structure.
4817 * @param[in, out] matcher
4818 * Pointer to flow matcher.
4819 * @parm[in, out] dev_flow
4820 * Pointer to the dev_flow.
4822 * pointer to error structure.
4825 * 0 on success otherwise -errno and errno is set.
4828 flow_dv_matcher_register(struct rte_eth_dev *dev,
4829 struct mlx5_flow_dv_matcher *matcher,
4830 struct mlx5_flow *dev_flow,
4831 struct rte_flow_error *error)
4833 struct mlx5_priv *priv = dev->data->dev_private;
4834 struct mlx5_ibv_shared *sh = priv->sh;
4835 struct mlx5_flow_dv_matcher *cache_matcher;
4836 struct mlx5dv_flow_matcher_attr dv_attr = {
4837 .type = IBV_FLOW_ATTR_NORMAL,
4838 .match_mask = (void *)&matcher->mask,
4840 struct mlx5_flow_tbl_resource *tbl = NULL;
4842 /* Lookup from cache. */
4843 LIST_FOREACH(cache_matcher, &sh->matchers, next) {
4844 if (matcher->crc == cache_matcher->crc &&
4845 matcher->priority == cache_matcher->priority &&
4846 matcher->egress == cache_matcher->egress &&
4847 matcher->group == cache_matcher->group &&
4848 matcher->transfer == cache_matcher->transfer &&
4849 !memcmp((const void *)matcher->mask.buf,
4850 (const void *)cache_matcher->mask.buf,
4851 cache_matcher->mask.size)) {
4853 "priority %hd use %s matcher %p: refcnt %d++",
4854 cache_matcher->priority,
4855 cache_matcher->egress ? "tx" : "rx",
4856 (void *)cache_matcher,
4857 rte_atomic32_read(&cache_matcher->refcnt));
4858 rte_atomic32_inc(&cache_matcher->refcnt);
4859 dev_flow->dv.matcher = cache_matcher;
4863 /* Register new matcher. */
4864 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
4866 return rte_flow_error_set(error, ENOMEM,
4867 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4868 "cannot allocate matcher memory");
4869 tbl = flow_dv_tbl_resource_get(dev, matcher->group,
4870 matcher->egress, matcher->transfer,
4873 rte_free(cache_matcher);
4874 return rte_flow_error_set(error, ENOMEM,
4875 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4876 NULL, "cannot create table");
4878 *cache_matcher = *matcher;
4879 dv_attr.match_criteria_enable =
4880 flow_dv_matcher_enable(cache_matcher->mask.buf);
4881 dv_attr.priority = matcher->priority;
4882 if (matcher->egress)
4883 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
4884 cache_matcher->matcher_object =
4885 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
4886 if (!cache_matcher->matcher_object) {
4887 rte_free(cache_matcher);
4888 #ifdef HAVE_MLX5DV_DR
4889 flow_dv_tbl_resource_release(tbl);
4891 return rte_flow_error_set(error, ENOMEM,
4892 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4893 NULL, "cannot create matcher");
4895 rte_atomic32_inc(&cache_matcher->refcnt);
4896 LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
4897 dev_flow->dv.matcher = cache_matcher;
4898 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
4899 cache_matcher->priority,
4900 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
4901 rte_atomic32_read(&cache_matcher->refcnt));
4902 rte_atomic32_inc(&tbl->refcnt);
4907 * Find existing tag resource or create and register a new one.
4909 * @param dev[in, out]
4910 * Pointer to rte_eth_dev structure.
4911 * @param[in, out] resource
4912 * Pointer to tag resource.
4913 * @parm[in, out] dev_flow
4914 * Pointer to the dev_flow.
4916 * pointer to error structure.
4919 * 0 on success otherwise -errno and errno is set.
4922 flow_dv_tag_resource_register
4923 (struct rte_eth_dev *dev,
4924 struct mlx5_flow_dv_tag_resource *resource,
4925 struct mlx5_flow *dev_flow,
4926 struct rte_flow_error *error)
4928 struct mlx5_priv *priv = dev->data->dev_private;
4929 struct mlx5_ibv_shared *sh = priv->sh;
4930 struct mlx5_flow_dv_tag_resource *cache_resource;
4932 /* Lookup a matching resource from cache. */
4933 LIST_FOREACH(cache_resource, &sh->tags, next) {
4934 if (resource->tag == cache_resource->tag) {
4935 DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
4936 (void *)cache_resource,
4937 rte_atomic32_read(&cache_resource->refcnt));
4938 rte_atomic32_inc(&cache_resource->refcnt);
4939 dev_flow->flow->tag_resource = cache_resource;
4943 /* Register new resource. */
4944 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
4945 if (!cache_resource)
4946 return rte_flow_error_set(error, ENOMEM,
4947 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4948 "cannot allocate resource memory");
4949 *cache_resource = *resource;
4950 cache_resource->action = mlx5_glue->dv_create_flow_action_tag
4952 if (!cache_resource->action) {
4953 rte_free(cache_resource);
4954 return rte_flow_error_set(error, ENOMEM,
4955 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4956 NULL, "cannot create action");
4958 rte_atomic32_init(&cache_resource->refcnt);
4959 rte_atomic32_inc(&cache_resource->refcnt);
4960 LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
4961 dev_flow->flow->tag_resource = cache_resource;
4962 DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
4963 (void *)cache_resource,
4964 rte_atomic32_read(&cache_resource->refcnt));
4972 * Pointer to Ethernet device.
4974 * Pointer to mlx5_flow.
4977 * 1 while a reference on it exists, 0 when freed.
4980 flow_dv_tag_release(struct rte_eth_dev *dev,
4981 struct mlx5_flow_dv_tag_resource *tag)
4984 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
4985 dev->data->port_id, (void *)tag,
4986 rte_atomic32_read(&tag->refcnt));
4987 if (rte_atomic32_dec_and_test(&tag->refcnt)) {
4988 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
4989 LIST_REMOVE(tag, next);
4990 DRV_LOG(DEBUG, "port %u tag %p: removed",
4991 dev->data->port_id, (void *)tag);
4999 * Translate port ID action to vport.
5002 * Pointer to rte_eth_dev structure.
5004 * Pointer to the port ID action.
5005 * @param[out] dst_port_id
5006 * The target port ID.
5008 * Pointer to the error structure.
5011 * 0 on success, a negative errno value otherwise and rte_errno is set.
5014 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
5015 const struct rte_flow_action *action,
5016 uint32_t *dst_port_id,
5017 struct rte_flow_error *error)
5022 const struct rte_flow_action_port_id *conf =
5023 (const struct rte_flow_action_port_id *)action->conf;
5025 port = conf->original ? dev->data->port_id : conf->id;
5026 ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
5028 return rte_flow_error_set(error, -ret,
5029 RTE_FLOW_ERROR_TYPE_ACTION,
5031 "No eswitch info was found for port");
5032 *dst_port_id = port_id;
5037 * Fill the flow with DV spec.
5040 * Pointer to rte_eth_dev structure.
5041 * @param[in, out] dev_flow
5042 * Pointer to the sub flow.
5044 * Pointer to the flow attributes.
5046 * Pointer to the list of items.
5047 * @param[in] actions
5048 * Pointer to the list of actions.
5050 * Pointer to the error structure.
5053 * 0 on success, a negative errno value otherwise and rte_errno is set.
5056 flow_dv_translate(struct rte_eth_dev *dev,
5057 struct mlx5_flow *dev_flow,
5058 const struct rte_flow_attr *attr,
5059 const struct rte_flow_item items[],
5060 const struct rte_flow_action actions[],
5061 struct rte_flow_error *error)
5063 struct mlx5_priv *priv = dev->data->dev_private;
5064 struct rte_flow *flow = dev_flow->flow;
5065 uint64_t item_flags = 0;
5066 uint64_t last_item = 0;
5067 uint64_t action_flags = 0;
5068 uint64_t priority = attr->priority;
5069 struct mlx5_flow_dv_matcher matcher = {
5071 .size = sizeof(matcher.mask.buf),
5075 bool actions_end = false;
5076 struct mlx5_flow_dv_modify_hdr_resource res = {
5077 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
5078 MLX5DV_FLOW_TABLE_TYPE_NIC_RX
5080 union flow_dv_attr flow_attr = { .attr = 0 };
5081 struct mlx5_flow_dv_tag_resource tag_resource;
5082 uint32_t modify_action_position = UINT32_MAX;
5083 void *match_mask = matcher.mask.buf;
5084 void *match_value = dev_flow->dv.value.buf;
5085 uint8_t next_protocol = 0xff;
5086 struct rte_vlan_hdr vlan = { 0 };
5087 bool vlan_inherited = false;
5090 flow->group = attr->group;
5092 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
5093 if (priority == MLX5_FLOW_PRIO_RSVD)
5094 priority = priv->config.flow_prio - 1;
5095 for (; !actions_end ; actions++) {
5096 const struct rte_flow_action_queue *queue;
5097 const struct rte_flow_action_rss *rss;
5098 const struct rte_flow_action *action = actions;
5099 const struct rte_flow_action_count *count = action->conf;
5100 const uint8_t *rss_key;
5101 const struct rte_flow_action_jump *jump_data;
5102 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
5103 struct mlx5_flow_tbl_resource *tbl;
5104 uint32_t port_id = 0;
5105 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
5107 switch (actions->type) {
5108 case RTE_FLOW_ACTION_TYPE_VOID:
5110 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5111 if (flow_dv_translate_action_port_id(dev, action,
5114 port_id_resource.port_id = port_id;
5115 if (flow_dv_port_id_action_resource_register
5116 (dev, &port_id_resource, dev_flow, error))
5118 dev_flow->dv.actions[actions_n++] =
5119 dev_flow->dv.port_id_action->action;
5120 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5122 case RTE_FLOW_ACTION_TYPE_FLAG:
5124 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
5125 if (!flow->tag_resource)
5126 if (flow_dv_tag_resource_register
5127 (dev, &tag_resource, dev_flow, error))
5129 dev_flow->dv.actions[actions_n++] =
5130 flow->tag_resource->action;
5131 action_flags |= MLX5_FLOW_ACTION_FLAG;
5133 case RTE_FLOW_ACTION_TYPE_MARK:
5134 tag_resource.tag = mlx5_flow_mark_set
5135 (((const struct rte_flow_action_mark *)
5136 (actions->conf))->id);
5137 if (!flow->tag_resource)
5138 if (flow_dv_tag_resource_register
5139 (dev, &tag_resource, dev_flow, error))
5141 dev_flow->dv.actions[actions_n++] =
5142 flow->tag_resource->action;
5143 action_flags |= MLX5_FLOW_ACTION_MARK;
5145 case RTE_FLOW_ACTION_TYPE_DROP:
5146 action_flags |= MLX5_FLOW_ACTION_DROP;
5148 case RTE_FLOW_ACTION_TYPE_QUEUE:
5149 queue = actions->conf;
5150 flow->rss.queue_num = 1;
5151 (*flow->queue)[0] = queue->index;
5152 action_flags |= MLX5_FLOW_ACTION_QUEUE;
5154 case RTE_FLOW_ACTION_TYPE_RSS:
5155 rss = actions->conf;
5157 memcpy((*flow->queue), rss->queue,
5158 rss->queue_num * sizeof(uint16_t));
5159 flow->rss.queue_num = rss->queue_num;
5160 /* NULL RSS key indicates default RSS key. */
5161 rss_key = !rss->key ? rss_hash_default_key : rss->key;
5162 memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
5163 /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
5164 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
5165 flow->rss.level = rss->level;
5166 action_flags |= MLX5_FLOW_ACTION_RSS;
5168 case RTE_FLOW_ACTION_TYPE_COUNT:
5169 if (!priv->config.devx) {
5170 rte_errno = ENOTSUP;
5173 flow->counter = flow_dv_counter_alloc(dev,
5177 if (flow->counter == NULL)
5179 dev_flow->dv.actions[actions_n++] =
5180 flow->counter->action;
5181 action_flags |= MLX5_FLOW_ACTION_COUNT;
5184 if (rte_errno == ENOTSUP)
5185 return rte_flow_error_set
5187 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5189 "count action not supported");
5191 return rte_flow_error_set
5193 RTE_FLOW_ERROR_TYPE_ACTION,
5195 "cannot create counter"
5198 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5199 dev_flow->dv.actions[actions_n++] =
5200 priv->sh->pop_vlan_action;
5201 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5203 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5204 if (!vlan_inherited) {
5205 flow_dev_get_vlan_info_from_items(items, &vlan);
5206 vlan_inherited = true;
5208 vlan.eth_proto = rte_be_to_cpu_16
5209 ((((const struct rte_flow_action_of_push_vlan *)
5210 actions->conf)->ethertype));
5211 if (flow_dv_create_action_push_vlan
5212 (dev, attr, &vlan, dev_flow, error))
5214 dev_flow->dv.actions[actions_n++] =
5215 dev_flow->dv.push_vlan_res->action;
5216 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5218 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5219 if (!vlan_inherited) {
5220 flow_dev_get_vlan_info_from_items(items, &vlan);
5221 vlan_inherited = true;
5224 ((const struct rte_flow_action_of_set_vlan_pcp *)
5225 actions->conf)->vlan_pcp;
5226 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
5227 vlan.vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
5228 vlan.vlan_tci |= vlan_tci;
5229 /* Push VLAN command will use this value */
5231 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5232 if (!vlan_inherited) {
5233 flow_dev_get_vlan_info_from_items(items, &vlan);
5234 vlan_inherited = true;
5236 vlan.vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
5237 vlan.vlan_tci |= rte_be_to_cpu_16
5238 (((const struct rte_flow_action_of_set_vlan_vid *)
5239 actions->conf)->vlan_vid);
5240 /* Push VLAN command will use this value */
5242 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5243 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5244 if (flow_dv_create_action_l2_encap(dev, actions,
5249 dev_flow->dv.actions[actions_n++] =
5250 dev_flow->dv.encap_decap->verbs_action;
5251 action_flags |= actions->type ==
5252 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
5253 MLX5_FLOW_ACTION_VXLAN_ENCAP :
5254 MLX5_FLOW_ACTION_NVGRE_ENCAP;
5256 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5257 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5258 if (flow_dv_create_action_l2_decap(dev, dev_flow,
5262 dev_flow->dv.actions[actions_n++] =
5263 dev_flow->dv.encap_decap->verbs_action;
5264 action_flags |= actions->type ==
5265 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
5266 MLX5_FLOW_ACTION_VXLAN_DECAP :
5267 MLX5_FLOW_ACTION_NVGRE_DECAP;
5269 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5270 /* Handle encap with preceding decap. */
5271 if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
5272 if (flow_dv_create_action_raw_encap
5273 (dev, actions, dev_flow, attr, error))
5275 dev_flow->dv.actions[actions_n++] =
5276 dev_flow->dv.encap_decap->verbs_action;
5278 /* Handle encap without preceding decap. */
5279 if (flow_dv_create_action_l2_encap
5280 (dev, actions, dev_flow, attr->transfer,
5283 dev_flow->dv.actions[actions_n++] =
5284 dev_flow->dv.encap_decap->verbs_action;
5286 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
5288 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5289 /* Check if this decap is followed by encap. */
5290 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
5291 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
5294 /* Handle decap only if it isn't followed by encap. */
5295 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5296 if (flow_dv_create_action_l2_decap
5297 (dev, dev_flow, attr->transfer, error))
5299 dev_flow->dv.actions[actions_n++] =
5300 dev_flow->dv.encap_decap->verbs_action;
5302 /* If decap is followed by encap, handle it at encap. */
5303 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
5305 case RTE_FLOW_ACTION_TYPE_JUMP:
5306 jump_data = action->conf;
5307 tbl = flow_dv_tbl_resource_get(dev, jump_data->group,
5309 attr->transfer, error);
5311 return rte_flow_error_set
5313 RTE_FLOW_ERROR_TYPE_ACTION,
5315 "cannot create jump action.");
5316 jump_tbl_resource.tbl = tbl;
5317 if (flow_dv_jump_tbl_resource_register
5318 (dev, &jump_tbl_resource, dev_flow, error)) {
5319 flow_dv_tbl_resource_release(tbl);
5320 return rte_flow_error_set
5322 RTE_FLOW_ERROR_TYPE_ACTION,
5324 "cannot create jump action.");
5326 dev_flow->dv.actions[actions_n++] =
5327 dev_flow->dv.jump->action;
5328 action_flags |= MLX5_FLOW_ACTION_JUMP;
5330 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5331 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5332 if (flow_dv_convert_action_modify_mac(&res, actions,
5335 action_flags |= actions->type ==
5336 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5337 MLX5_FLOW_ACTION_SET_MAC_SRC :
5338 MLX5_FLOW_ACTION_SET_MAC_DST;
5340 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5341 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5342 if (flow_dv_convert_action_modify_ipv4(&res, actions,
5345 action_flags |= actions->type ==
5346 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5347 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5348 MLX5_FLOW_ACTION_SET_IPV4_DST;
5350 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5351 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5352 if (flow_dv_convert_action_modify_ipv6(&res, actions,
5355 action_flags |= actions->type ==
5356 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5357 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5358 MLX5_FLOW_ACTION_SET_IPV6_DST;
5360 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5361 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5362 if (flow_dv_convert_action_modify_tp(&res, actions,
5366 action_flags |= actions->type ==
5367 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5368 MLX5_FLOW_ACTION_SET_TP_SRC :
5369 MLX5_FLOW_ACTION_SET_TP_DST;
5371 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5372 if (flow_dv_convert_action_modify_dec_ttl(&res, items,
5376 action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
5378 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5379 if (flow_dv_convert_action_modify_ttl(&res, actions,
5383 action_flags |= MLX5_FLOW_ACTION_SET_TTL;
5385 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5386 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5387 if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
5390 action_flags |= actions->type ==
5391 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5392 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5393 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5396 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5397 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5398 if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
5401 action_flags |= actions->type ==
5402 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5403 MLX5_FLOW_ACTION_INC_TCP_ACK :
5404 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5406 case RTE_FLOW_ACTION_TYPE_END:
5408 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
5409 /* create modify action if needed. */
5410 if (flow_dv_modify_hdr_resource_register
5415 dev_flow->dv.actions[modify_action_position] =
5416 dev_flow->dv.modify_hdr->verbs_action;
5422 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
5423 modify_action_position == UINT32_MAX)
5424 modify_action_position = actions_n++;
5426 dev_flow->dv.actions_n = actions_n;
5427 flow->actions = action_flags;
5428 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5429 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5431 switch (items->type) {
5432 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5433 flow_dv_translate_item_port_id(dev, match_mask,
5434 match_value, items);
5435 last_item = MLX5_FLOW_ITEM_PORT_ID;
5437 case RTE_FLOW_ITEM_TYPE_ETH:
5438 flow_dv_translate_item_eth(match_mask, match_value,
5440 matcher.priority = MLX5_PRIORITY_MAP_L2;
5441 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5442 MLX5_FLOW_LAYER_OUTER_L2;
5444 case RTE_FLOW_ITEM_TYPE_VLAN:
5445 flow_dv_translate_item_vlan(dev_flow,
5446 match_mask, match_value,
5448 matcher.priority = MLX5_PRIORITY_MAP_L2;
5449 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
5450 MLX5_FLOW_LAYER_INNER_VLAN) :
5451 (MLX5_FLOW_LAYER_OUTER_L2 |
5452 MLX5_FLOW_LAYER_OUTER_VLAN);
5454 case RTE_FLOW_ITEM_TYPE_IPV4:
5455 mlx5_flow_tunnel_ip_check(items, next_protocol,
5456 &item_flags, &tunnel);
5457 flow_dv_translate_item_ipv4(match_mask, match_value,
5458 items, tunnel, attr->group);
5459 matcher.priority = MLX5_PRIORITY_MAP_L3;
5460 dev_flow->dv.hash_fields |=
5461 mlx5_flow_hashfields_adjust
5463 MLX5_IPV4_LAYER_TYPES,
5464 MLX5_IPV4_IBV_RX_HASH);
5465 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5466 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5467 if (items->mask != NULL &&
5468 ((const struct rte_flow_item_ipv4 *)
5469 items->mask)->hdr.next_proto_id) {
5471 ((const struct rte_flow_item_ipv4 *)
5472 (items->spec))->hdr.next_proto_id;
5474 ((const struct rte_flow_item_ipv4 *)
5475 (items->mask))->hdr.next_proto_id;
5477 /* Reset for inner layer. */
5478 next_protocol = 0xff;
5481 case RTE_FLOW_ITEM_TYPE_IPV6:
5482 mlx5_flow_tunnel_ip_check(items, next_protocol,
5483 &item_flags, &tunnel);
5484 flow_dv_translate_item_ipv6(match_mask, match_value,
5485 items, tunnel, attr->group);
5486 matcher.priority = MLX5_PRIORITY_MAP_L3;
5487 dev_flow->dv.hash_fields |=
5488 mlx5_flow_hashfields_adjust
5490 MLX5_IPV6_LAYER_TYPES,
5491 MLX5_IPV6_IBV_RX_HASH);
5492 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5493 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5494 if (items->mask != NULL &&
5495 ((const struct rte_flow_item_ipv6 *)
5496 items->mask)->hdr.proto) {
5498 ((const struct rte_flow_item_ipv6 *)
5499 items->spec)->hdr.proto;
5501 ((const struct rte_flow_item_ipv6 *)
5502 items->mask)->hdr.proto;
5504 /* Reset for inner layer. */
5505 next_protocol = 0xff;
5508 case RTE_FLOW_ITEM_TYPE_TCP:
5509 flow_dv_translate_item_tcp(match_mask, match_value,
5511 matcher.priority = MLX5_PRIORITY_MAP_L4;
5512 dev_flow->dv.hash_fields |=
5513 mlx5_flow_hashfields_adjust
5514 (dev_flow, tunnel, ETH_RSS_TCP,
5515 IBV_RX_HASH_SRC_PORT_TCP |
5516 IBV_RX_HASH_DST_PORT_TCP);
5517 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5518 MLX5_FLOW_LAYER_OUTER_L4_TCP;
5520 case RTE_FLOW_ITEM_TYPE_UDP:
5521 flow_dv_translate_item_udp(match_mask, match_value,
5523 matcher.priority = MLX5_PRIORITY_MAP_L4;
5524 dev_flow->dv.hash_fields |=
5525 mlx5_flow_hashfields_adjust
5526 (dev_flow, tunnel, ETH_RSS_UDP,
5527 IBV_RX_HASH_SRC_PORT_UDP |
5528 IBV_RX_HASH_DST_PORT_UDP);
5529 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5530 MLX5_FLOW_LAYER_OUTER_L4_UDP;
5532 case RTE_FLOW_ITEM_TYPE_GRE:
5533 flow_dv_translate_item_gre(match_mask, match_value,
5535 last_item = MLX5_FLOW_LAYER_GRE;
5537 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5538 flow_dv_translate_item_gre_key(match_mask,
5539 match_value, items);
5540 last_item = MLX5_FLOW_LAYER_GRE_KEY;
5542 case RTE_FLOW_ITEM_TYPE_NVGRE:
5543 flow_dv_translate_item_nvgre(match_mask, match_value,
5545 last_item = MLX5_FLOW_LAYER_GRE;
5547 case RTE_FLOW_ITEM_TYPE_VXLAN:
5548 flow_dv_translate_item_vxlan(match_mask, match_value,
5550 last_item = MLX5_FLOW_LAYER_VXLAN;
5552 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5553 flow_dv_translate_item_vxlan(match_mask, match_value,
5555 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5557 case RTE_FLOW_ITEM_TYPE_MPLS:
5558 flow_dv_translate_item_mpls(match_mask, match_value,
5559 items, last_item, tunnel);
5560 last_item = MLX5_FLOW_LAYER_MPLS;
5562 case RTE_FLOW_ITEM_TYPE_META:
5563 flow_dv_translate_item_meta(match_mask, match_value,
5565 last_item = MLX5_FLOW_ITEM_METADATA;
5567 case RTE_FLOW_ITEM_TYPE_ICMP:
5568 flow_dv_translate_item_icmp(match_mask, match_value,
5570 last_item = MLX5_FLOW_LAYER_ICMP;
5572 case RTE_FLOW_ITEM_TYPE_ICMP6:
5573 flow_dv_translate_item_icmp6(match_mask, match_value,
5575 last_item = MLX5_FLOW_LAYER_ICMP6;
5580 item_flags |= last_item;
5583 * In case of ingress traffic when E-Switch mode is enabled,
5584 * we have two cases where we need to set the source port manually.
5585 * The first one, is in case of Nic steering rule, and the second is
5586 * E-Switch rule where no port_id item was found. In both cases
5587 * the source port is set according the current port in use.
5589 if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
5590 (priv->representor || priv->master)) {
5591 if (flow_dv_translate_item_port_id(dev, match_mask,
5595 assert(!flow_dv_check_valid_spec(matcher.mask.buf,
5596 dev_flow->dv.value.buf));
5597 dev_flow->layers = item_flags;
5598 /* Register matcher. */
5599 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
5601 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
5603 matcher.egress = attr->egress;
5604 matcher.group = attr->group;
5605 matcher.transfer = attr->transfer;
5606 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
5612 * Apply the flow to the NIC.
5615 * Pointer to the Ethernet device structure.
5616 * @param[in, out] flow
5617 * Pointer to flow structure.
5619 * Pointer to error structure.
5622 * 0 on success, a negative errno value otherwise and rte_errno is set.
5625 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5626 struct rte_flow_error *error)
5628 struct mlx5_flow_dv *dv;
5629 struct mlx5_flow *dev_flow;
5630 struct mlx5_priv *priv = dev->data->dev_private;
5634 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5637 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
5638 if (flow->transfer) {
5639 dv->actions[n++] = priv->sh->esw_drop_action;
5641 dv->hrxq = mlx5_hrxq_drop_new(dev);
5645 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5647 "cannot get drop hash queue");
5650 dv->actions[n++] = dv->hrxq->action;
5652 } else if (flow->actions &
5653 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
5654 struct mlx5_hrxq *hrxq;
5656 hrxq = mlx5_hrxq_get(dev, flow->key,
5657 MLX5_RSS_HASH_KEY_LEN,
5660 flow->rss.queue_num);
5662 hrxq = mlx5_hrxq_new
5663 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
5664 dv->hash_fields, (*flow->queue),
5665 flow->rss.queue_num,
5666 !!(dev_flow->layers &
5667 MLX5_FLOW_LAYER_TUNNEL));
5672 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5673 "cannot get hash queue");
5677 dv->actions[n++] = dv->hrxq->action;
5680 mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
5681 (void *)&dv->value, n,
5684 rte_flow_error_set(error, errno,
5685 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5687 "hardware refuses to create flow");
5690 if (priv->vmwa_context &&
5691 dev_flow->dv.vf_vlan.tag &&
5692 !dev_flow->dv.vf_vlan.created) {
5694 * The rule contains the VLAN pattern.
5695 * For VF we are going to create VLAN
5696 * interface to make hypervisor set correct
5697 * e-Switch vport context.
5699 mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
5704 err = rte_errno; /* Save rte_errno before cleanup. */
5705 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5706 struct mlx5_flow_dv *dv = &dev_flow->dv;
5708 if (flow->actions & MLX5_FLOW_ACTION_DROP)
5709 mlx5_hrxq_drop_release(dev);
5711 mlx5_hrxq_release(dev, dv->hrxq);
5714 if (dev_flow->dv.vf_vlan.tag &&
5715 dev_flow->dv.vf_vlan.created)
5716 mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5718 rte_errno = err; /* Restore rte_errno. */
5723 * Release the flow matcher.
5726 * Pointer to Ethernet device.
5728 * Pointer to mlx5_flow.
5731 * 1 while a reference on it exists, 0 when freed.
5734 flow_dv_matcher_release(struct rte_eth_dev *dev,
5735 struct mlx5_flow *flow)
5737 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
5738 struct mlx5_priv *priv = dev->data->dev_private;
5739 struct mlx5_ibv_shared *sh = priv->sh;
5740 struct mlx5_flow_tbl_resource *tbl;
5742 assert(matcher->matcher_object);
5743 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
5744 dev->data->port_id, (void *)matcher,
5745 rte_atomic32_read(&matcher->refcnt));
5746 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
5747 claim_zero(mlx5_glue->dv_destroy_flow_matcher
5748 (matcher->matcher_object));
5749 LIST_REMOVE(matcher, next);
5750 if (matcher->egress)
5751 tbl = &sh->tx_tbl[matcher->group];
5753 tbl = &sh->rx_tbl[matcher->group];
5754 flow_dv_tbl_resource_release(tbl);
5756 DRV_LOG(DEBUG, "port %u matcher %p: removed",
5757 dev->data->port_id, (void *)matcher);
5764 * Release an encap/decap resource.
5767 * Pointer to mlx5_flow.
5770 * 1 while a reference on it exists, 0 when freed.
5773 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
5775 struct mlx5_flow_dv_encap_decap_resource *cache_resource =
5776 flow->dv.encap_decap;
5778 assert(cache_resource->verbs_action);
5779 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
5780 (void *)cache_resource,
5781 rte_atomic32_read(&cache_resource->refcnt));
5782 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5783 claim_zero(mlx5_glue->destroy_flow_action
5784 (cache_resource->verbs_action));
5785 LIST_REMOVE(cache_resource, next);
5786 rte_free(cache_resource);
5787 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
5788 (void *)cache_resource);
5795 * Release an jump to table action resource.
5798 * Pointer to mlx5_flow.
5801 * 1 while a reference on it exists, 0 when freed.
5804 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
5806 struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
5809 assert(cache_resource->action);
5810 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
5811 (void *)cache_resource,
5812 rte_atomic32_read(&cache_resource->refcnt));
5813 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5814 claim_zero(mlx5_glue->destroy_flow_action
5815 (cache_resource->action));
5816 LIST_REMOVE(cache_resource, next);
5817 flow_dv_tbl_resource_release(cache_resource->tbl);
5818 rte_free(cache_resource);
5819 DRV_LOG(DEBUG, "jump table resource %p: removed",
5820 (void *)cache_resource);
5827 * Release a modify-header resource.
5830 * Pointer to mlx5_flow.
5833 * 1 while a reference on it exists, 0 when freed.
5836 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
5838 struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
5839 flow->dv.modify_hdr;
5841 assert(cache_resource->verbs_action);
5842 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
5843 (void *)cache_resource,
5844 rte_atomic32_read(&cache_resource->refcnt));
5845 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5846 claim_zero(mlx5_glue->destroy_flow_action
5847 (cache_resource->verbs_action));
5848 LIST_REMOVE(cache_resource, next);
5849 rte_free(cache_resource);
5850 DRV_LOG(DEBUG, "modify-header resource %p: removed",
5851 (void *)cache_resource);
5858 * Release port ID action resource.
5861 * Pointer to mlx5_flow.
5864 * 1 while a reference on it exists, 0 when freed.
5867 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
5869 struct mlx5_flow_dv_port_id_action_resource *cache_resource =
5870 flow->dv.port_id_action;
5872 assert(cache_resource->action);
5873 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
5874 (void *)cache_resource,
5875 rte_atomic32_read(&cache_resource->refcnt));
5876 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5877 claim_zero(mlx5_glue->destroy_flow_action
5878 (cache_resource->action));
5879 LIST_REMOVE(cache_resource, next);
5880 rte_free(cache_resource);
5881 DRV_LOG(DEBUG, "port id action resource %p: removed",
5882 (void *)cache_resource);
5889 * Release push vlan action resource.
5892 * Pointer to mlx5_flow.
5895 * 1 while a reference on it exists, 0 when freed.
5898 flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow)
5900 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource =
5901 flow->dv.push_vlan_res;
5903 assert(cache_resource->action);
5904 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
5905 (void *)cache_resource,
5906 rte_atomic32_read(&cache_resource->refcnt));
5907 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5908 claim_zero(mlx5_glue->destroy_flow_action
5909 (cache_resource->action));
5910 LIST_REMOVE(cache_resource, next);
5911 rte_free(cache_resource);
5912 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
5913 (void *)cache_resource);
5920 * Remove the flow from the NIC but keeps it in memory.
5923 * Pointer to Ethernet device.
5924 * @param[in, out] flow
5925 * Pointer to flow structure.
5928 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5930 struct mlx5_flow_dv *dv;
5931 struct mlx5_flow *dev_flow;
5935 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5938 claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
5942 if (flow->actions & MLX5_FLOW_ACTION_DROP)
5943 mlx5_hrxq_drop_release(dev);
5945 mlx5_hrxq_release(dev, dv->hrxq);
5948 if (dev_flow->dv.vf_vlan.tag &&
5949 dev_flow->dv.vf_vlan.created)
5950 mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5955 * Remove the flow from the NIC and the memory.
5958 * Pointer to the Ethernet device structure.
5959 * @param[in, out] flow
5960 * Pointer to flow structure.
5963 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5965 struct mlx5_flow *dev_flow;
5969 flow_dv_remove(dev, flow);
5970 if (flow->counter) {
5971 flow_dv_counter_release(dev, flow->counter);
5972 flow->counter = NULL;
5974 if (flow->tag_resource) {
5975 flow_dv_tag_release(dev, flow->tag_resource);
5976 flow->tag_resource = NULL;
5978 while (!LIST_EMPTY(&flow->dev_flows)) {
5979 dev_flow = LIST_FIRST(&flow->dev_flows);
5980 LIST_REMOVE(dev_flow, next);
5981 if (dev_flow->dv.matcher)
5982 flow_dv_matcher_release(dev, dev_flow);
5983 if (dev_flow->dv.encap_decap)
5984 flow_dv_encap_decap_resource_release(dev_flow);
5985 if (dev_flow->dv.modify_hdr)
5986 flow_dv_modify_hdr_resource_release(dev_flow);
5987 if (dev_flow->dv.jump)
5988 flow_dv_jump_tbl_resource_release(dev_flow);
5989 if (dev_flow->dv.port_id_action)
5990 flow_dv_port_id_action_resource_release(dev_flow);
5991 if (dev_flow->dv.push_vlan_res)
5992 flow_dv_push_vlan_action_resource_release(dev_flow);
5998 * Query a dv flow rule for its statistics via devx.
6001 * Pointer to Ethernet device.
6003 * Pointer to the sub flow.
6005 * data retrieved by the query.
6007 * Perform verbose error reporting if not NULL.
6010 * 0 on success, a negative errno value otherwise and rte_errno is set.
6013 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
6014 void *data, struct rte_flow_error *error)
6016 struct mlx5_priv *priv = dev->data->dev_private;
6017 struct rte_flow_query_count *qc = data;
6019 if (!priv->config.devx)
6020 return rte_flow_error_set(error, ENOTSUP,
6021 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6023 "counters are not supported");
6024 if (flow->counter) {
6025 uint64_t pkts, bytes;
6026 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
6030 return rte_flow_error_set(error, -err,
6031 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6032 NULL, "cannot read counters");
6035 qc->hits = pkts - flow->counter->hits;
6036 qc->bytes = bytes - flow->counter->bytes;
6038 flow->counter->hits = pkts;
6039 flow->counter->bytes = bytes;
6043 return rte_flow_error_set(error, EINVAL,
6044 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6046 "counters are not available");
6052 * @see rte_flow_query()
6056 flow_dv_query(struct rte_eth_dev *dev,
6057 struct rte_flow *flow __rte_unused,
6058 const struct rte_flow_action *actions __rte_unused,
6059 void *data __rte_unused,
6060 struct rte_flow_error *error __rte_unused)
6064 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
6065 switch (actions->type) {
6066 case RTE_FLOW_ACTION_TYPE_VOID:
6068 case RTE_FLOW_ACTION_TYPE_COUNT:
6069 ret = flow_dv_query_count(dev, flow, data, error);
6072 return rte_flow_error_set(error, ENOTSUP,
6073 RTE_FLOW_ERROR_TYPE_ACTION,
6075 "action not supported");
6082 * Mutex-protected thunk to flow_dv_translate().
6085 flow_d_translate(struct rte_eth_dev *dev,
6086 struct mlx5_flow *dev_flow,
6087 const struct rte_flow_attr *attr,
6088 const struct rte_flow_item items[],
6089 const struct rte_flow_action actions[],
6090 struct rte_flow_error *error)
6094 flow_d_shared_lock(dev);
6095 ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
6096 flow_d_shared_unlock(dev);
6101 * Mutex-protected thunk to flow_dv_apply().
6104 flow_d_apply(struct rte_eth_dev *dev,
6105 struct rte_flow *flow,
6106 struct rte_flow_error *error)
6110 flow_d_shared_lock(dev);
6111 ret = flow_dv_apply(dev, flow, error);
6112 flow_d_shared_unlock(dev);
6117 * Mutex-protected thunk to flow_dv_remove().
6120 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
6122 flow_d_shared_lock(dev);
6123 flow_dv_remove(dev, flow);
6124 flow_d_shared_unlock(dev);
6128 * Mutex-protected thunk to flow_dv_destroy().
6131 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
6133 flow_d_shared_lock(dev);
6134 flow_dv_destroy(dev, flow);
6135 flow_d_shared_unlock(dev);
6138 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
6139 .validate = flow_dv_validate,
6140 .prepare = flow_dv_prepare,
6141 .translate = flow_d_translate,
6142 .apply = flow_d_apply,
6143 .remove = flow_d_remove,
6144 .destroy = flow_d_destroy,
6145 .query = flow_dv_query,
6148 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */