1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
31 #include "mlx5_defs.h"
32 #include "mlx5_glue.h"
33 #include "mlx5_flow.h"
35 #include "mlx5_rxtx.h"
37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
43 #ifndef HAVE_MLX5DV_DR_ESWITCH
44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
49 #ifndef HAVE_MLX5DV_DR
50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
53 /* VLAN header definitions */
54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
73 * Initialize flow attributes structure according to flow items' types.
76 * Pointer to item specification.
78 * Pointer to flow attributes structure.
81 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
83 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
85 case RTE_FLOW_ITEM_TYPE_IPV4:
88 case RTE_FLOW_ITEM_TYPE_IPV6:
91 case RTE_FLOW_ITEM_TYPE_UDP:
94 case RTE_FLOW_ITEM_TYPE_TCP:
104 struct field_modify_info {
105 uint32_t size; /* Size of field in protocol header, in bytes. */
106 uint32_t offset; /* Offset of field in protocol header, in bytes. */
107 enum mlx5_modification_field id;
110 struct field_modify_info modify_eth[] = {
111 {4, 0, MLX5_MODI_OUT_DMAC_47_16},
112 {2, 4, MLX5_MODI_OUT_DMAC_15_0},
113 {4, 6, MLX5_MODI_OUT_SMAC_47_16},
114 {2, 10, MLX5_MODI_OUT_SMAC_15_0},
118 struct field_modify_info modify_ipv4[] = {
119 {1, 8, MLX5_MODI_OUT_IPV4_TTL},
120 {4, 12, MLX5_MODI_OUT_SIPV4},
121 {4, 16, MLX5_MODI_OUT_DIPV4},
125 struct field_modify_info modify_ipv6[] = {
126 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
127 {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
128 {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
129 {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
130 {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
131 {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
132 {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
133 {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
134 {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
138 struct field_modify_info modify_udp[] = {
139 {2, 0, MLX5_MODI_OUT_UDP_SPORT},
140 {2, 2, MLX5_MODI_OUT_UDP_DPORT},
144 struct field_modify_info modify_tcp[] = {
145 {2, 0, MLX5_MODI_OUT_TCP_SPORT},
146 {2, 2, MLX5_MODI_OUT_TCP_DPORT},
147 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
148 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
153 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
154 uint8_t next_protocol, uint64_t *item_flags,
157 assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
158 item->type == RTE_FLOW_ITEM_TYPE_IPV6);
159 if (next_protocol == IPPROTO_IPIP) {
160 *item_flags |= MLX5_FLOW_LAYER_IPIP;
163 if (next_protocol == IPPROTO_IPV6) {
164 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
170 * Acquire the synchronizing object to protect multithreaded access
171 * to shared dv context. Lock occurs only if context is actually
172 * shared, i.e. we have multiport IB device and representors are
176 * Pointer to the rte_eth_dev structure.
179 flow_d_shared_lock(struct rte_eth_dev *dev)
181 struct mlx5_priv *priv = dev->data->dev_private;
182 struct mlx5_ibv_shared *sh = priv->sh;
184 if (sh->dv_refcnt > 1) {
187 ret = pthread_mutex_lock(&sh->dv_mutex);
194 flow_d_shared_unlock(struct rte_eth_dev *dev)
196 struct mlx5_priv *priv = dev->data->dev_private;
197 struct mlx5_ibv_shared *sh = priv->sh;
199 if (sh->dv_refcnt > 1) {
202 ret = pthread_mutex_unlock(&sh->dv_mutex);
209 * Convert modify-header action to DV specification.
212 * Pointer to item specification.
214 * Pointer to field modification information.
215 * @param[in,out] resource
216 * Pointer to the modify-header resource.
218 * Type of modification.
220 * Pointer to the error structure.
223 * 0 on success, a negative errno value otherwise and rte_errno is set.
226 flow_dv_convert_modify_action(struct rte_flow_item *item,
227 struct field_modify_info *field,
228 struct mlx5_flow_dv_modify_hdr_resource *resource,
230 struct rte_flow_error *error)
232 uint32_t i = resource->actions_num;
233 struct mlx5_modification_cmd *actions = resource->actions;
234 const uint8_t *spec = item->spec;
235 const uint8_t *mask = item->mask;
238 while (field->size) {
240 /* Generate modify command for each mask segment. */
241 memcpy(&set, &mask[field->offset], field->size);
243 if (i >= MLX5_MODIFY_NUM)
244 return rte_flow_error_set(error, EINVAL,
245 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
246 "too many items to modify");
247 actions[i].action_type = type;
248 actions[i].field = field->id;
249 actions[i].length = field->size ==
250 4 ? 0 : field->size * 8;
251 rte_memcpy(&actions[i].data[4 - field->size],
252 &spec[field->offset], field->size);
253 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
256 if (resource->actions_num != i)
257 resource->actions_num = i;
260 if (!resource->actions_num)
261 return rte_flow_error_set(error, EINVAL,
262 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
263 "invalid modification flow item");
268 * Convert modify-header set IPv4 address action to DV specification.
270 * @param[in,out] resource
271 * Pointer to the modify-header resource.
273 * Pointer to action specification.
275 * Pointer to the error structure.
278 * 0 on success, a negative errno value otherwise and rte_errno is set.
281 flow_dv_convert_action_modify_ipv4
282 (struct mlx5_flow_dv_modify_hdr_resource *resource,
283 const struct rte_flow_action *action,
284 struct rte_flow_error *error)
286 const struct rte_flow_action_set_ipv4 *conf =
287 (const struct rte_flow_action_set_ipv4 *)(action->conf);
288 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
289 struct rte_flow_item_ipv4 ipv4;
290 struct rte_flow_item_ipv4 ipv4_mask;
292 memset(&ipv4, 0, sizeof(ipv4));
293 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
294 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
295 ipv4.hdr.src_addr = conf->ipv4_addr;
296 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
298 ipv4.hdr.dst_addr = conf->ipv4_addr;
299 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
302 item.mask = &ipv4_mask;
303 return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
304 MLX5_MODIFICATION_TYPE_SET, error);
308 * Convert modify-header set IPv6 address action to DV specification.
310 * @param[in,out] resource
311 * Pointer to the modify-header resource.
313 * Pointer to action specification.
315 * Pointer to the error structure.
318 * 0 on success, a negative errno value otherwise and rte_errno is set.
321 flow_dv_convert_action_modify_ipv6
322 (struct mlx5_flow_dv_modify_hdr_resource *resource,
323 const struct rte_flow_action *action,
324 struct rte_flow_error *error)
326 const struct rte_flow_action_set_ipv6 *conf =
327 (const struct rte_flow_action_set_ipv6 *)(action->conf);
328 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
329 struct rte_flow_item_ipv6 ipv6;
330 struct rte_flow_item_ipv6 ipv6_mask;
332 memset(&ipv6, 0, sizeof(ipv6));
333 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
334 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
335 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
336 sizeof(ipv6.hdr.src_addr));
337 memcpy(&ipv6_mask.hdr.src_addr,
338 &rte_flow_item_ipv6_mask.hdr.src_addr,
339 sizeof(ipv6.hdr.src_addr));
341 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
342 sizeof(ipv6.hdr.dst_addr));
343 memcpy(&ipv6_mask.hdr.dst_addr,
344 &rte_flow_item_ipv6_mask.hdr.dst_addr,
345 sizeof(ipv6.hdr.dst_addr));
348 item.mask = &ipv6_mask;
349 return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
350 MLX5_MODIFICATION_TYPE_SET, error);
354 * Convert modify-header set MAC address action to DV specification.
356 * @param[in,out] resource
357 * Pointer to the modify-header resource.
359 * Pointer to action specification.
361 * Pointer to the error structure.
364 * 0 on success, a negative errno value otherwise and rte_errno is set.
367 flow_dv_convert_action_modify_mac
368 (struct mlx5_flow_dv_modify_hdr_resource *resource,
369 const struct rte_flow_action *action,
370 struct rte_flow_error *error)
372 const struct rte_flow_action_set_mac *conf =
373 (const struct rte_flow_action_set_mac *)(action->conf);
374 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
375 struct rte_flow_item_eth eth;
376 struct rte_flow_item_eth eth_mask;
378 memset(ð, 0, sizeof(eth));
379 memset(ð_mask, 0, sizeof(eth_mask));
380 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
381 memcpy(ð.src.addr_bytes, &conf->mac_addr,
382 sizeof(eth.src.addr_bytes));
383 memcpy(ð_mask.src.addr_bytes,
384 &rte_flow_item_eth_mask.src.addr_bytes,
385 sizeof(eth_mask.src.addr_bytes));
387 memcpy(ð.dst.addr_bytes, &conf->mac_addr,
388 sizeof(eth.dst.addr_bytes));
389 memcpy(ð_mask.dst.addr_bytes,
390 &rte_flow_item_eth_mask.dst.addr_bytes,
391 sizeof(eth_mask.dst.addr_bytes));
394 item.mask = ð_mask;
395 return flow_dv_convert_modify_action(&item, modify_eth, resource,
396 MLX5_MODIFICATION_TYPE_SET, error);
400 * Convert modify-header set TP action to DV specification.
402 * @param[in,out] resource
403 * Pointer to the modify-header resource.
405 * Pointer to action specification.
407 * Pointer to rte_flow_item objects list.
409 * Pointer to flow attributes structure.
411 * Pointer to the error structure.
414 * 0 on success, a negative errno value otherwise and rte_errno is set.
417 flow_dv_convert_action_modify_tp
418 (struct mlx5_flow_dv_modify_hdr_resource *resource,
419 const struct rte_flow_action *action,
420 const struct rte_flow_item *items,
421 union flow_dv_attr *attr,
422 struct rte_flow_error *error)
424 const struct rte_flow_action_set_tp *conf =
425 (const struct rte_flow_action_set_tp *)(action->conf);
426 struct rte_flow_item item;
427 struct rte_flow_item_udp udp;
428 struct rte_flow_item_udp udp_mask;
429 struct rte_flow_item_tcp tcp;
430 struct rte_flow_item_tcp tcp_mask;
431 struct field_modify_info *field;
434 flow_dv_attr_init(items, attr);
436 memset(&udp, 0, sizeof(udp));
437 memset(&udp_mask, 0, sizeof(udp_mask));
438 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
439 udp.hdr.src_port = conf->port;
440 udp_mask.hdr.src_port =
441 rte_flow_item_udp_mask.hdr.src_port;
443 udp.hdr.dst_port = conf->port;
444 udp_mask.hdr.dst_port =
445 rte_flow_item_udp_mask.hdr.dst_port;
447 item.type = RTE_FLOW_ITEM_TYPE_UDP;
449 item.mask = &udp_mask;
453 memset(&tcp, 0, sizeof(tcp));
454 memset(&tcp_mask, 0, sizeof(tcp_mask));
455 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
456 tcp.hdr.src_port = conf->port;
457 tcp_mask.hdr.src_port =
458 rte_flow_item_tcp_mask.hdr.src_port;
460 tcp.hdr.dst_port = conf->port;
461 tcp_mask.hdr.dst_port =
462 rte_flow_item_tcp_mask.hdr.dst_port;
464 item.type = RTE_FLOW_ITEM_TYPE_TCP;
466 item.mask = &tcp_mask;
469 return flow_dv_convert_modify_action(&item, field, resource,
470 MLX5_MODIFICATION_TYPE_SET, error);
474 * Convert modify-header set TTL action to DV specification.
476 * @param[in,out] resource
477 * Pointer to the modify-header resource.
479 * Pointer to action specification.
481 * Pointer to rte_flow_item objects list.
483 * Pointer to flow attributes structure.
485 * Pointer to the error structure.
488 * 0 on success, a negative errno value otherwise and rte_errno is set.
491 flow_dv_convert_action_modify_ttl
492 (struct mlx5_flow_dv_modify_hdr_resource *resource,
493 const struct rte_flow_action *action,
494 const struct rte_flow_item *items,
495 union flow_dv_attr *attr,
496 struct rte_flow_error *error)
498 const struct rte_flow_action_set_ttl *conf =
499 (const struct rte_flow_action_set_ttl *)(action->conf);
500 struct rte_flow_item item;
501 struct rte_flow_item_ipv4 ipv4;
502 struct rte_flow_item_ipv4 ipv4_mask;
503 struct rte_flow_item_ipv6 ipv6;
504 struct rte_flow_item_ipv6 ipv6_mask;
505 struct field_modify_info *field;
508 flow_dv_attr_init(items, attr);
510 memset(&ipv4, 0, sizeof(ipv4));
511 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
512 ipv4.hdr.time_to_live = conf->ttl_value;
513 ipv4_mask.hdr.time_to_live = 0xFF;
514 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
516 item.mask = &ipv4_mask;
520 memset(&ipv6, 0, sizeof(ipv6));
521 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
522 ipv6.hdr.hop_limits = conf->ttl_value;
523 ipv6_mask.hdr.hop_limits = 0xFF;
524 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
526 item.mask = &ipv6_mask;
529 return flow_dv_convert_modify_action(&item, field, resource,
530 MLX5_MODIFICATION_TYPE_SET, error);
534 * Convert modify-header decrement TTL action to DV specification.
536 * @param[in,out] resource
537 * Pointer to the modify-header resource.
539 * Pointer to action specification.
541 * Pointer to rte_flow_item objects list.
543 * Pointer to flow attributes structure.
545 * Pointer to the error structure.
548 * 0 on success, a negative errno value otherwise and rte_errno is set.
551 flow_dv_convert_action_modify_dec_ttl
552 (struct mlx5_flow_dv_modify_hdr_resource *resource,
553 const struct rte_flow_item *items,
554 union flow_dv_attr *attr,
555 struct rte_flow_error *error)
557 struct rte_flow_item item;
558 struct rte_flow_item_ipv4 ipv4;
559 struct rte_flow_item_ipv4 ipv4_mask;
560 struct rte_flow_item_ipv6 ipv6;
561 struct rte_flow_item_ipv6 ipv6_mask;
562 struct field_modify_info *field;
565 flow_dv_attr_init(items, attr);
567 memset(&ipv4, 0, sizeof(ipv4));
568 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
569 ipv4.hdr.time_to_live = 0xFF;
570 ipv4_mask.hdr.time_to_live = 0xFF;
571 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
573 item.mask = &ipv4_mask;
577 memset(&ipv6, 0, sizeof(ipv6));
578 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
579 ipv6.hdr.hop_limits = 0xFF;
580 ipv6_mask.hdr.hop_limits = 0xFF;
581 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
583 item.mask = &ipv6_mask;
586 return flow_dv_convert_modify_action(&item, field, resource,
587 MLX5_MODIFICATION_TYPE_ADD, error);
591 * Convert modify-header increment/decrement TCP Sequence number
592 * to DV specification.
594 * @param[in,out] resource
595 * Pointer to the modify-header resource.
597 * Pointer to action specification.
599 * Pointer to the error structure.
602 * 0 on success, a negative errno value otherwise and rte_errno is set.
605 flow_dv_convert_action_modify_tcp_seq
606 (struct mlx5_flow_dv_modify_hdr_resource *resource,
607 const struct rte_flow_action *action,
608 struct rte_flow_error *error)
610 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
611 uint64_t value = rte_be_to_cpu_32(*conf);
612 struct rte_flow_item item;
613 struct rte_flow_item_tcp tcp;
614 struct rte_flow_item_tcp tcp_mask;
616 memset(&tcp, 0, sizeof(tcp));
617 memset(&tcp_mask, 0, sizeof(tcp_mask));
618 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
620 * The HW has no decrement operation, only increment operation.
621 * To simulate decrement X from Y using increment operation
622 * we need to add UINT32_MAX X times to Y.
623 * Each adding of UINT32_MAX decrements Y by 1.
626 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
627 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
628 item.type = RTE_FLOW_ITEM_TYPE_TCP;
630 item.mask = &tcp_mask;
631 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
632 MLX5_MODIFICATION_TYPE_ADD, error);
636 * Convert modify-header increment/decrement TCP Acknowledgment number
637 * to DV specification.
639 * @param[in,out] resource
640 * Pointer to the modify-header resource.
642 * Pointer to action specification.
644 * Pointer to the error structure.
647 * 0 on success, a negative errno value otherwise and rte_errno is set.
650 flow_dv_convert_action_modify_tcp_ack
651 (struct mlx5_flow_dv_modify_hdr_resource *resource,
652 const struct rte_flow_action *action,
653 struct rte_flow_error *error)
655 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
656 uint64_t value = rte_be_to_cpu_32(*conf);
657 struct rte_flow_item item;
658 struct rte_flow_item_tcp tcp;
659 struct rte_flow_item_tcp tcp_mask;
661 memset(&tcp, 0, sizeof(tcp));
662 memset(&tcp_mask, 0, sizeof(tcp_mask));
663 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
665 * The HW has no decrement operation, only increment operation.
666 * To simulate decrement X from Y using increment operation
667 * we need to add UINT32_MAX X times to Y.
668 * Each adding of UINT32_MAX decrements Y by 1.
671 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
672 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
673 item.type = RTE_FLOW_ITEM_TYPE_TCP;
675 item.mask = &tcp_mask;
676 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
677 MLX5_MODIFICATION_TYPE_ADD, error);
681 * Validate META item.
684 * Pointer to the rte_eth_dev structure.
686 * Item specification.
688 * Attributes of flow that includes this item.
690 * Pointer to error structure.
693 * 0 on success, a negative errno value otherwise and rte_errno is set.
696 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
697 const struct rte_flow_item *item,
698 const struct rte_flow_attr *attr,
699 struct rte_flow_error *error)
701 const struct rte_flow_item_meta *spec = item->spec;
702 const struct rte_flow_item_meta *mask = item->mask;
703 const struct rte_flow_item_meta nic_mask = {
704 .data = RTE_BE32(UINT32_MAX)
707 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
709 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
710 return rte_flow_error_set(error, EPERM,
711 RTE_FLOW_ERROR_TYPE_ITEM,
713 "match on metadata offload "
714 "configuration is off for this port");
716 return rte_flow_error_set(error, EINVAL,
717 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
719 "data cannot be empty");
721 return rte_flow_error_set(error, EINVAL,
722 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
724 "data cannot be zero");
726 mask = &rte_flow_item_meta_mask;
727 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
728 (const uint8_t *)&nic_mask,
729 sizeof(struct rte_flow_item_meta),
734 return rte_flow_error_set(error, ENOTSUP,
735 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
737 "pattern not supported for ingress");
742 * Validate vport item.
745 * Pointer to the rte_eth_dev structure.
747 * Item specification.
749 * Attributes of flow that includes this item.
750 * @param[in] item_flags
751 * Bit-fields that holds the items detected until now.
753 * Pointer to error structure.
756 * 0 on success, a negative errno value otherwise and rte_errno is set.
759 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
760 const struct rte_flow_item *item,
761 const struct rte_flow_attr *attr,
763 struct rte_flow_error *error)
765 const struct rte_flow_item_port_id *spec = item->spec;
766 const struct rte_flow_item_port_id *mask = item->mask;
767 const struct rte_flow_item_port_id switch_mask = {
770 uint16_t esw_domain_id;
771 uint16_t item_port_esw_domain_id;
775 return rte_flow_error_set(error, EINVAL,
776 RTE_FLOW_ERROR_TYPE_ITEM,
778 "match on port id is valid only"
779 " when transfer flag is enabled");
780 if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
781 return rte_flow_error_set(error, ENOTSUP,
782 RTE_FLOW_ERROR_TYPE_ITEM, item,
783 "multiple source ports are not"
787 if (mask->id != 0xffffffff)
788 return rte_flow_error_set(error, ENOTSUP,
789 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
791 "no support for partial mask on"
793 ret = mlx5_flow_item_acceptable
794 (item, (const uint8_t *)mask,
795 (const uint8_t *)&rte_flow_item_port_id_mask,
796 sizeof(struct rte_flow_item_port_id),
802 ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
805 return rte_flow_error_set(error, -ret,
806 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
807 "failed to obtain E-Switch info for"
809 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
810 &esw_domain_id, NULL);
812 return rte_flow_error_set(error, -ret,
813 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
815 "failed to obtain E-Switch info");
816 if (item_port_esw_domain_id != esw_domain_id)
817 return rte_flow_error_set(error, -ret,
818 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
819 "cannot match on a port from a"
820 " different E-Switch");
825 * Validate the pop VLAN action.
828 * Pointer to the rte_eth_dev structure.
829 * @param[in] action_flags
830 * Holds the actions detected until now.
832 * Pointer to the pop vlan action.
833 * @param[in] item_flags
834 * The items found in this flow rule.
836 * Pointer to flow attributes.
838 * Pointer to error structure.
841 * 0 on success, a negative errno value otherwise and rte_errno is set.
844 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
845 uint64_t action_flags,
846 const struct rte_flow_action *action,
848 const struct rte_flow_attr *attr,
849 struct rte_flow_error *error)
851 struct mlx5_priv *priv = dev->data->dev_private;
855 if (!priv->sh->pop_vlan_action)
856 return rte_flow_error_set(error, ENOTSUP,
857 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
859 "pop vlan action is not supported");
861 * Check for inconsistencies:
862 * fail strip_vlan in a flow that matches packets without VLAN tags.
863 * fail strip_vlan in a flow that matches packets without explicitly a
864 * matching on VLAN tag ?
866 if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN)
867 return rte_flow_error_set(error, ENOTSUP,
868 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
870 "no support for multiple vlan pop "
872 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
873 return rte_flow_error_set(error, ENOTSUP,
874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
876 "cannot pop vlan without a "
877 "match on (outer) vlan in the flow");
882 * Get VLAN default info from vlan match info.
885 * Pointer to the rte_eth_dev structure.
887 * the list of item specifications.
889 * pointer VLAN info to fill to.
891 * Pointer to error structure.
894 * 0 on success, a negative errno value otherwise and rte_errno is set.
897 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
898 struct rte_vlan_hdr *vlan)
900 const struct rte_flow_item_vlan nic_mask = {
901 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
902 MLX5DV_FLOW_VLAN_VID_MASK),
903 .inner_type = RTE_BE16(0xffff),
908 for (; items->type != RTE_FLOW_ITEM_TYPE_END &&
909 items->type != RTE_FLOW_ITEM_TYPE_VLAN; items++)
911 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
912 const struct rte_flow_item_vlan *vlan_m = items->mask;
913 const struct rte_flow_item_vlan *vlan_v = items->spec;
917 /* Only full match values are accepted */
918 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
919 MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
920 vlan->vlan_tci &= MLX5DV_FLOW_VLAN_PCP_MASK;
922 rte_be_to_cpu_16(vlan_v->tci &
923 MLX5DV_FLOW_VLAN_PCP_MASK_BE);
925 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
926 MLX5DV_FLOW_VLAN_VID_MASK_BE) {
927 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
929 rte_be_to_cpu_16(vlan_v->tci &
930 MLX5DV_FLOW_VLAN_VID_MASK_BE);
932 if (vlan_m->inner_type == nic_mask.inner_type)
933 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
939 * Validate the push VLAN action.
941 * @param[in] action_flags
942 * Holds the actions detected until now.
944 * Pointer to the encap action.
946 * Pointer to flow attributes
948 * Pointer to error structure.
951 * 0 on success, a negative errno value otherwise and rte_errno is set.
954 flow_dv_validate_action_push_vlan(uint64_t action_flags,
955 const struct rte_flow_action *action,
956 const struct rte_flow_attr *attr,
957 struct rte_flow_error *error)
959 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
961 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
962 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
963 return rte_flow_error_set(error, EINVAL,
964 RTE_FLOW_ERROR_TYPE_ACTION, action,
965 "invalid vlan ethertype");
967 (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN))
968 return rte_flow_error_set(error, ENOTSUP,
969 RTE_FLOW_ERROR_TYPE_ACTION, action,
970 "no support for multiple VLAN "
977 * Validate count action.
982 * Pointer to error structure.
985 * 0 on success, a negative errno value otherwise and rte_errno is set.
988 flow_dv_validate_action_count(struct rte_eth_dev *dev,
989 struct rte_flow_error *error)
991 struct mlx5_priv *priv = dev->data->dev_private;
993 if (!priv->config.devx)
995 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
999 return rte_flow_error_set
1001 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1003 "count action not supported");
1007 * Validate the L2 encap action.
1009 * @param[in] action_flags
1010 * Holds the actions detected until now.
1012 * Pointer to the encap action.
1014 * Pointer to flow attributes
1016 * Pointer to error structure.
1019 * 0 on success, a negative errno value otherwise and rte_errno is set.
1022 flow_dv_validate_action_l2_encap(uint64_t action_flags,
1023 const struct rte_flow_action *action,
1024 const struct rte_flow_attr *attr,
1025 struct rte_flow_error *error)
1027 if (!(action->conf))
1028 return rte_flow_error_set(error, EINVAL,
1029 RTE_FLOW_ERROR_TYPE_ACTION, action,
1030 "configuration cannot be null");
1031 if (action_flags & MLX5_FLOW_ACTION_DROP)
1032 return rte_flow_error_set(error, EINVAL,
1033 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1034 "can't drop and encap in same flow");
1035 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1036 return rte_flow_error_set(error, EINVAL,
1037 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1038 "can only have a single encap or"
1039 " decap action in a flow");
1040 if (!attr->transfer && attr->ingress)
1041 return rte_flow_error_set(error, ENOTSUP,
1042 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1044 "encap action not supported for "
1050 * Validate the L2 decap action.
1052 * @param[in] action_flags
1053 * Holds the actions detected until now.
1055 * Pointer to flow attributes
1057 * Pointer to error structure.
1060 * 0 on success, a negative errno value otherwise and rte_errno is set.
1063 flow_dv_validate_action_l2_decap(uint64_t action_flags,
1064 const struct rte_flow_attr *attr,
1065 struct rte_flow_error *error)
1067 if (action_flags & MLX5_FLOW_ACTION_DROP)
1068 return rte_flow_error_set(error, EINVAL,
1069 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1070 "can't drop and decap in same flow");
1071 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
1072 return rte_flow_error_set(error, EINVAL,
1073 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1074 "can only have a single encap or"
1075 " decap action in a flow");
1076 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1077 return rte_flow_error_set(error, EINVAL,
1078 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1079 "can't have decap action after"
1082 return rte_flow_error_set(error, ENOTSUP,
1083 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1085 "decap action not supported for "
1091 * Validate the raw encap action.
1093 * @param[in] action_flags
1094 * Holds the actions detected until now.
1096 * Pointer to the encap action.
1098 * Pointer to flow attributes
1100 * Pointer to error structure.
1103 * 0 on success, a negative errno value otherwise and rte_errno is set.
1106 flow_dv_validate_action_raw_encap(uint64_t action_flags,
1107 const struct rte_flow_action *action,
1108 const struct rte_flow_attr *attr,
1109 struct rte_flow_error *error)
1111 const struct rte_flow_action_raw_encap *raw_encap =
1112 (const struct rte_flow_action_raw_encap *)action->conf;
1113 if (!(action->conf))
1114 return rte_flow_error_set(error, EINVAL,
1115 RTE_FLOW_ERROR_TYPE_ACTION, action,
1116 "configuration cannot be null");
1117 if (action_flags & MLX5_FLOW_ACTION_DROP)
1118 return rte_flow_error_set(error, EINVAL,
1119 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1120 "can't drop and encap in same flow");
1121 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1122 return rte_flow_error_set(error, EINVAL,
1123 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1124 "can only have a single encap"
1125 " action in a flow");
1126 /* encap without preceding decap is not supported for ingress */
1127 if (!attr->transfer && attr->ingress &&
1128 !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
1129 return rte_flow_error_set(error, ENOTSUP,
1130 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1132 "encap action not supported for "
1134 if (!raw_encap->size || !raw_encap->data)
1135 return rte_flow_error_set(error, EINVAL,
1136 RTE_FLOW_ERROR_TYPE_ACTION, action,
1137 "raw encap data cannot be empty");
1142 * Validate the raw decap action.
1144 * @param[in] action_flags
1145 * Holds the actions detected until now.
1147 * Pointer to the encap action.
1149 * Pointer to flow attributes
1151 * Pointer to error structure.
1154 * 0 on success, a negative errno value otherwise and rte_errno is set.
1157 flow_dv_validate_action_raw_decap(uint64_t action_flags,
1158 const struct rte_flow_action *action,
1159 const struct rte_flow_attr *attr,
1160 struct rte_flow_error *error)
1162 if (action_flags & MLX5_FLOW_ACTION_DROP)
1163 return rte_flow_error_set(error, EINVAL,
1164 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1165 "can't drop and decap in same flow");
1166 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1167 return rte_flow_error_set(error, EINVAL,
1168 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1169 "can't have encap action before"
1171 if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
1172 return rte_flow_error_set(error, EINVAL,
1173 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1174 "can only have a single decap"
1175 " action in a flow");
1176 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
1177 return rte_flow_error_set(error, EINVAL,
1178 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1179 "can't have decap action after"
1181 /* decap action is valid on egress only if it is followed by encap */
1183 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1184 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1187 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1188 return rte_flow_error_set
1190 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1191 NULL, "decap action not supported"
1198 * Find existing encap/decap resource or create and register a new one.
1200 * @param dev[in, out]
1201 * Pointer to rte_eth_dev structure.
1202 * @param[in, out] resource
1203 * Pointer to encap/decap resource.
1204 * @parm[in, out] dev_flow
1205 * Pointer to the dev_flow.
1207 * pointer to error structure.
1210 * 0 on success otherwise -errno and errno is set.
1213 flow_dv_encap_decap_resource_register
1214 (struct rte_eth_dev *dev,
1215 struct mlx5_flow_dv_encap_decap_resource *resource,
1216 struct mlx5_flow *dev_flow,
1217 struct rte_flow_error *error)
1219 struct mlx5_priv *priv = dev->data->dev_private;
1220 struct mlx5_ibv_shared *sh = priv->sh;
1221 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1222 struct rte_flow *flow = dev_flow->flow;
1223 struct mlx5dv_dr_domain *domain;
1225 resource->flags = flow->group ? 0 : 1;
1226 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1227 domain = sh->fdb_domain;
1228 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1229 domain = sh->rx_domain;
1231 domain = sh->tx_domain;
1233 /* Lookup a matching resource from cache. */
1234 LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1235 if (resource->reformat_type == cache_resource->reformat_type &&
1236 resource->ft_type == cache_resource->ft_type &&
1237 resource->flags == cache_resource->flags &&
1238 resource->size == cache_resource->size &&
1239 !memcmp((const void *)resource->buf,
1240 (const void *)cache_resource->buf,
1242 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1243 (void *)cache_resource,
1244 rte_atomic32_read(&cache_resource->refcnt));
1245 rte_atomic32_inc(&cache_resource->refcnt);
1246 dev_flow->dv.encap_decap = cache_resource;
1250 /* Register new encap/decap resource. */
1251 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1252 if (!cache_resource)
1253 return rte_flow_error_set(error, ENOMEM,
1254 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1255 "cannot allocate resource memory");
1256 *cache_resource = *resource;
1257 cache_resource->verbs_action =
1258 mlx5_glue->dv_create_flow_action_packet_reformat
1259 (sh->ctx, cache_resource->reformat_type,
1260 cache_resource->ft_type, domain, cache_resource->flags,
1261 cache_resource->size,
1262 (cache_resource->size ? cache_resource->buf : NULL));
1263 if (!cache_resource->verbs_action) {
1264 rte_free(cache_resource);
1265 return rte_flow_error_set(error, ENOMEM,
1266 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1267 NULL, "cannot create action");
1269 rte_atomic32_init(&cache_resource->refcnt);
1270 rte_atomic32_inc(&cache_resource->refcnt);
1271 LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1272 dev_flow->dv.encap_decap = cache_resource;
1273 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1274 (void *)cache_resource,
1275 rte_atomic32_read(&cache_resource->refcnt));
1280 * Find existing table jump resource or create and register a new one.
1282 * @param dev[in, out]
1283 * Pointer to rte_eth_dev structure.
1284 * @param[in, out] resource
1285 * Pointer to jump table resource.
1286 * @parm[in, out] dev_flow
1287 * Pointer to the dev_flow.
1289 * pointer to error structure.
1292 * 0 on success otherwise -errno and errno is set.
1295 flow_dv_jump_tbl_resource_register
1296 (struct rte_eth_dev *dev,
1297 struct mlx5_flow_dv_jump_tbl_resource *resource,
1298 struct mlx5_flow *dev_flow,
1299 struct rte_flow_error *error)
1301 struct mlx5_priv *priv = dev->data->dev_private;
1302 struct mlx5_ibv_shared *sh = priv->sh;
1303 struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1305 /* Lookup a matching resource from cache. */
1306 LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1307 if (resource->tbl == cache_resource->tbl) {
1308 DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1309 (void *)cache_resource,
1310 rte_atomic32_read(&cache_resource->refcnt));
1311 rte_atomic32_inc(&cache_resource->refcnt);
1312 dev_flow->dv.jump = cache_resource;
1316 /* Register new jump table resource. */
1317 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1318 if (!cache_resource)
1319 return rte_flow_error_set(error, ENOMEM,
1320 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1321 "cannot allocate resource memory");
1322 *cache_resource = *resource;
1323 cache_resource->action =
1324 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1325 (resource->tbl->obj);
1326 if (!cache_resource->action) {
1327 rte_free(cache_resource);
1328 return rte_flow_error_set(error, ENOMEM,
1329 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1330 NULL, "cannot create action");
1332 rte_atomic32_init(&cache_resource->refcnt);
1333 rte_atomic32_inc(&cache_resource->refcnt);
1334 LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1335 dev_flow->dv.jump = cache_resource;
1336 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
1337 (void *)cache_resource,
1338 rte_atomic32_read(&cache_resource->refcnt));
1343 * Find existing table port ID resource or create and register a new one.
1345 * @param dev[in, out]
1346 * Pointer to rte_eth_dev structure.
1347 * @param[in, out] resource
1348 * Pointer to port ID action resource.
1349 * @parm[in, out] dev_flow
1350 * Pointer to the dev_flow.
1352 * pointer to error structure.
1355 * 0 on success otherwise -errno and errno is set.
1358 flow_dv_port_id_action_resource_register
1359 (struct rte_eth_dev *dev,
1360 struct mlx5_flow_dv_port_id_action_resource *resource,
1361 struct mlx5_flow *dev_flow,
1362 struct rte_flow_error *error)
1364 struct mlx5_priv *priv = dev->data->dev_private;
1365 struct mlx5_ibv_shared *sh = priv->sh;
1366 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1368 /* Lookup a matching resource from cache. */
1369 LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1370 if (resource->port_id == cache_resource->port_id) {
1371 DRV_LOG(DEBUG, "port id action resource resource %p: "
1373 (void *)cache_resource,
1374 rte_atomic32_read(&cache_resource->refcnt));
1375 rte_atomic32_inc(&cache_resource->refcnt);
1376 dev_flow->dv.port_id_action = cache_resource;
1380 /* Register new port id action resource. */
1381 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1382 if (!cache_resource)
1383 return rte_flow_error_set(error, ENOMEM,
1384 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1385 "cannot allocate resource memory");
1386 *cache_resource = *resource;
1387 cache_resource->action =
1388 mlx5_glue->dr_create_flow_action_dest_vport
1389 (priv->sh->fdb_domain, resource->port_id);
1390 if (!cache_resource->action) {
1391 rte_free(cache_resource);
1392 return rte_flow_error_set(error, ENOMEM,
1393 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1394 NULL, "cannot create action");
1396 rte_atomic32_init(&cache_resource->refcnt);
1397 rte_atomic32_inc(&cache_resource->refcnt);
1398 LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1399 dev_flow->dv.port_id_action = cache_resource;
1400 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1401 (void *)cache_resource,
1402 rte_atomic32_read(&cache_resource->refcnt));
1407 * Find existing push vlan resource or create and register a new one.
1409 * @param dev[in, out]
1410 * Pointer to rte_eth_dev structure.
1411 * @param[in, out] resource
1412 * Pointer to port ID action resource.
1413 * @parm[in, out] dev_flow
1414 * Pointer to the dev_flow.
1416 * pointer to error structure.
1419 * 0 on success otherwise -errno and errno is set.
1422 flow_dv_push_vlan_action_resource_register
1423 (struct rte_eth_dev *dev,
1424 struct mlx5_flow_dv_push_vlan_action_resource *resource,
1425 struct mlx5_flow *dev_flow,
1426 struct rte_flow_error *error)
1428 struct mlx5_priv *priv = dev->data->dev_private;
1429 struct mlx5_ibv_shared *sh = priv->sh;
1430 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
1431 struct mlx5dv_dr_domain *domain;
1433 /* Lookup a matching resource from cache. */
1434 LIST_FOREACH(cache_resource, &sh->push_vlan_action_list, next) {
1435 if (resource->vlan_tag == cache_resource->vlan_tag &&
1436 resource->ft_type == cache_resource->ft_type) {
1437 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
1439 (void *)cache_resource,
1440 rte_atomic32_read(&cache_resource->refcnt));
1441 rte_atomic32_inc(&cache_resource->refcnt);
1442 dev_flow->dv.push_vlan_res = cache_resource;
1446 /* Register new push_vlan action resource. */
1447 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1448 if (!cache_resource)
1449 return rte_flow_error_set(error, ENOMEM,
1450 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1451 "cannot allocate resource memory");
1452 *cache_resource = *resource;
1453 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1454 domain = sh->fdb_domain;
1455 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1456 domain = sh->rx_domain;
1458 domain = sh->tx_domain;
1459 cache_resource->action =
1460 mlx5_glue->dr_create_flow_action_push_vlan(domain,
1461 resource->vlan_tag);
1462 if (!cache_resource->action) {
1463 rte_free(cache_resource);
1464 return rte_flow_error_set(error, ENOMEM,
1465 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1466 NULL, "cannot create action");
1468 rte_atomic32_init(&cache_resource->refcnt);
1469 rte_atomic32_inc(&cache_resource->refcnt);
1470 LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next);
1471 dev_flow->dv.push_vlan_res = cache_resource;
1472 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
1473 (void *)cache_resource,
1474 rte_atomic32_read(&cache_resource->refcnt));
1478 * Get the size of specific rte_flow_item_type
1480 * @param[in] item_type
1481 * Tested rte_flow_item_type.
1484 * sizeof struct item_type, 0 if void or irrelevant.
1487 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1491 switch (item_type) {
1492 case RTE_FLOW_ITEM_TYPE_ETH:
1493 retval = sizeof(struct rte_flow_item_eth);
1495 case RTE_FLOW_ITEM_TYPE_VLAN:
1496 retval = sizeof(struct rte_flow_item_vlan);
1498 case RTE_FLOW_ITEM_TYPE_IPV4:
1499 retval = sizeof(struct rte_flow_item_ipv4);
1501 case RTE_FLOW_ITEM_TYPE_IPV6:
1502 retval = sizeof(struct rte_flow_item_ipv6);
1504 case RTE_FLOW_ITEM_TYPE_UDP:
1505 retval = sizeof(struct rte_flow_item_udp);
1507 case RTE_FLOW_ITEM_TYPE_TCP:
1508 retval = sizeof(struct rte_flow_item_tcp);
1510 case RTE_FLOW_ITEM_TYPE_VXLAN:
1511 retval = sizeof(struct rte_flow_item_vxlan);
1513 case RTE_FLOW_ITEM_TYPE_GRE:
1514 retval = sizeof(struct rte_flow_item_gre);
1516 case RTE_FLOW_ITEM_TYPE_NVGRE:
1517 retval = sizeof(struct rte_flow_item_nvgre);
1519 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1520 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1522 case RTE_FLOW_ITEM_TYPE_MPLS:
1523 retval = sizeof(struct rte_flow_item_mpls);
1525 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1533 #define MLX5_ENCAP_IPV4_VERSION 0x40
1534 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
1535 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
1536 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
1537 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
1538 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
1539 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
1542 * Convert the encap action data from list of rte_flow_item to raw buffer
1545 * Pointer to rte_flow_item objects list.
1547 * Pointer to the output buffer.
1549 * Pointer to the output buffer size.
1551 * Pointer to the error structure.
1554 * 0 on success, a negative errno value otherwise and rte_errno is set.
1557 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1558 size_t *size, struct rte_flow_error *error)
1560 struct rte_ether_hdr *eth = NULL;
1561 struct rte_vlan_hdr *vlan = NULL;
1562 struct rte_ipv4_hdr *ipv4 = NULL;
1563 struct rte_ipv6_hdr *ipv6 = NULL;
1564 struct rte_udp_hdr *udp = NULL;
1565 struct rte_vxlan_hdr *vxlan = NULL;
1566 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1567 struct rte_gre_hdr *gre = NULL;
1569 size_t temp_size = 0;
1572 return rte_flow_error_set(error, EINVAL,
1573 RTE_FLOW_ERROR_TYPE_ACTION,
1574 NULL, "invalid empty data");
1575 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1576 len = flow_dv_get_item_len(items->type);
1577 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1578 return rte_flow_error_set(error, EINVAL,
1579 RTE_FLOW_ERROR_TYPE_ACTION,
1580 (void *)items->type,
1581 "items total size is too big"
1582 " for encap action");
1583 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1584 switch (items->type) {
1585 case RTE_FLOW_ITEM_TYPE_ETH:
1586 eth = (struct rte_ether_hdr *)&buf[temp_size];
1588 case RTE_FLOW_ITEM_TYPE_VLAN:
1589 vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1591 return rte_flow_error_set(error, EINVAL,
1592 RTE_FLOW_ERROR_TYPE_ACTION,
1593 (void *)items->type,
1594 "eth header not found");
1595 if (!eth->ether_type)
1596 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1598 case RTE_FLOW_ITEM_TYPE_IPV4:
1599 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1601 return rte_flow_error_set(error, EINVAL,
1602 RTE_FLOW_ERROR_TYPE_ACTION,
1603 (void *)items->type,
1604 "neither eth nor vlan"
1606 if (vlan && !vlan->eth_proto)
1607 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1608 else if (eth && !eth->ether_type)
1609 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1610 if (!ipv4->version_ihl)
1611 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1612 MLX5_ENCAP_IPV4_IHL_MIN;
1613 if (!ipv4->time_to_live)
1614 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1616 case RTE_FLOW_ITEM_TYPE_IPV6:
1617 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1619 return rte_flow_error_set(error, EINVAL,
1620 RTE_FLOW_ERROR_TYPE_ACTION,
1621 (void *)items->type,
1622 "neither eth nor vlan"
1624 if (vlan && !vlan->eth_proto)
1625 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1626 else if (eth && !eth->ether_type)
1627 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1628 if (!ipv6->vtc_flow)
1630 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1631 if (!ipv6->hop_limits)
1632 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1634 case RTE_FLOW_ITEM_TYPE_UDP:
1635 udp = (struct rte_udp_hdr *)&buf[temp_size];
1637 return rte_flow_error_set(error, EINVAL,
1638 RTE_FLOW_ERROR_TYPE_ACTION,
1639 (void *)items->type,
1640 "ip header not found");
1641 if (ipv4 && !ipv4->next_proto_id)
1642 ipv4->next_proto_id = IPPROTO_UDP;
1643 else if (ipv6 && !ipv6->proto)
1644 ipv6->proto = IPPROTO_UDP;
1646 case RTE_FLOW_ITEM_TYPE_VXLAN:
1647 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1649 return rte_flow_error_set(error, EINVAL,
1650 RTE_FLOW_ERROR_TYPE_ACTION,
1651 (void *)items->type,
1652 "udp header not found");
1654 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1655 if (!vxlan->vx_flags)
1657 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1659 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1660 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1662 return rte_flow_error_set(error, EINVAL,
1663 RTE_FLOW_ERROR_TYPE_ACTION,
1664 (void *)items->type,
1665 "udp header not found");
1666 if (!vxlan_gpe->proto)
1667 return rte_flow_error_set(error, EINVAL,
1668 RTE_FLOW_ERROR_TYPE_ACTION,
1669 (void *)items->type,
1670 "next protocol not found");
1673 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1674 if (!vxlan_gpe->vx_flags)
1675 vxlan_gpe->vx_flags =
1676 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1678 case RTE_FLOW_ITEM_TYPE_GRE:
1679 case RTE_FLOW_ITEM_TYPE_NVGRE:
1680 gre = (struct rte_gre_hdr *)&buf[temp_size];
1682 return rte_flow_error_set(error, EINVAL,
1683 RTE_FLOW_ERROR_TYPE_ACTION,
1684 (void *)items->type,
1685 "next protocol not found");
1687 return rte_flow_error_set(error, EINVAL,
1688 RTE_FLOW_ERROR_TYPE_ACTION,
1689 (void *)items->type,
1690 "ip header not found");
1691 if (ipv4 && !ipv4->next_proto_id)
1692 ipv4->next_proto_id = IPPROTO_GRE;
1693 else if (ipv6 && !ipv6->proto)
1694 ipv6->proto = IPPROTO_GRE;
1696 case RTE_FLOW_ITEM_TYPE_VOID:
1699 return rte_flow_error_set(error, EINVAL,
1700 RTE_FLOW_ERROR_TYPE_ACTION,
1701 (void *)items->type,
1702 "unsupported item type");
1712 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1714 struct rte_ether_hdr *eth = NULL;
1715 struct rte_vlan_hdr *vlan = NULL;
1716 struct rte_ipv6_hdr *ipv6 = NULL;
1717 struct rte_udp_hdr *udp = NULL;
1721 eth = (struct rte_ether_hdr *)data;
1722 next_hdr = (char *)(eth + 1);
1723 proto = RTE_BE16(eth->ether_type);
1726 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1727 vlan = (struct rte_vlan_hdr *)next_hdr;
1728 proto = RTE_BE16(vlan->eth_proto);
1729 next_hdr += sizeof(struct rte_vlan_hdr);
1732 /* HW calculates IPv4 csum. no need to proceed */
1733 if (proto == RTE_ETHER_TYPE_IPV4)
1736 /* non IPv4/IPv6 header. not supported */
1737 if (proto != RTE_ETHER_TYPE_IPV6) {
1738 return rte_flow_error_set(error, ENOTSUP,
1739 RTE_FLOW_ERROR_TYPE_ACTION,
1740 NULL, "Cannot offload non IPv4/IPv6");
1743 ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1745 /* ignore non UDP */
1746 if (ipv6->proto != IPPROTO_UDP)
1749 udp = (struct rte_udp_hdr *)(ipv6 + 1);
1750 udp->dgram_cksum = 0;
1756 * Convert L2 encap action to DV specification.
1759 * Pointer to rte_eth_dev structure.
1761 * Pointer to action structure.
1762 * @param[in, out] dev_flow
1763 * Pointer to the mlx5_flow.
1764 * @param[in] transfer
1765 * Mark if the flow is E-Switch flow.
1767 * Pointer to the error structure.
1770 * 0 on success, a negative errno value otherwise and rte_errno is set.
1773 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1774 const struct rte_flow_action *action,
1775 struct mlx5_flow *dev_flow,
1777 struct rte_flow_error *error)
1779 const struct rte_flow_item *encap_data;
1780 const struct rte_flow_action_raw_encap *raw_encap_data;
1781 struct mlx5_flow_dv_encap_decap_resource res = {
1783 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1784 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1785 MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1788 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1790 (const struct rte_flow_action_raw_encap *)action->conf;
1791 res.size = raw_encap_data->size;
1792 memcpy(res.buf, raw_encap_data->data, res.size);
1793 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1796 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1798 ((const struct rte_flow_action_vxlan_encap *)
1799 action->conf)->definition;
1802 ((const struct rte_flow_action_nvgre_encap *)
1803 action->conf)->definition;
1804 if (flow_dv_convert_encap_data(encap_data, res.buf,
1808 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1809 return rte_flow_error_set(error, EINVAL,
1810 RTE_FLOW_ERROR_TYPE_ACTION,
1811 NULL, "can't create L2 encap action");
1816 * Convert L2 decap action to DV specification.
1819 * Pointer to rte_eth_dev structure.
1820 * @param[in, out] dev_flow
1821 * Pointer to the mlx5_flow.
1822 * @param[in] transfer
1823 * Mark if the flow is E-Switch flow.
1825 * Pointer to the error structure.
1828 * 0 on success, a negative errno value otherwise and rte_errno is set.
1831 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1832 struct mlx5_flow *dev_flow,
1834 struct rte_flow_error *error)
1836 struct mlx5_flow_dv_encap_decap_resource res = {
1839 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1840 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1841 MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1844 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1845 return rte_flow_error_set(error, EINVAL,
1846 RTE_FLOW_ERROR_TYPE_ACTION,
1847 NULL, "can't create L2 decap action");
1852 * Convert raw decap/encap (L3 tunnel) action to DV specification.
1855 * Pointer to rte_eth_dev structure.
1857 * Pointer to action structure.
1858 * @param[in, out] dev_flow
1859 * Pointer to the mlx5_flow.
1861 * Pointer to the flow attributes.
1863 * Pointer to the error structure.
1866 * 0 on success, a negative errno value otherwise and rte_errno is set.
1869 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1870 const struct rte_flow_action *action,
1871 struct mlx5_flow *dev_flow,
1872 const struct rte_flow_attr *attr,
1873 struct rte_flow_error *error)
1875 const struct rte_flow_action_raw_encap *encap_data;
1876 struct mlx5_flow_dv_encap_decap_resource res;
1878 encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1879 res.size = encap_data->size;
1880 memcpy(res.buf, encap_data->data, res.size);
1881 res.reformat_type = attr->egress ?
1882 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1883 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1885 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1887 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1888 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1889 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1890 return rte_flow_error_set(error, EINVAL,
1891 RTE_FLOW_ERROR_TYPE_ACTION,
1892 NULL, "can't create encap action");
1897 * Create action push VLAN.
1900 * Pointer to rte_eth_dev structure.
1901 * @param[in] vlan_tag
1902 * the vlan tag to push to the Ethernet header.
1903 * @param[in, out] dev_flow
1904 * Pointer to the mlx5_flow.
1906 * Pointer to the flow attributes.
1908 * Pointer to the error structure.
1911 * 0 on success, a negative errno value otherwise and rte_errno is set.
1914 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
1915 const struct rte_flow_attr *attr,
1916 const struct rte_vlan_hdr *vlan,
1917 struct mlx5_flow *dev_flow,
1918 struct rte_flow_error *error)
1920 struct mlx5_flow_dv_push_vlan_action_resource res;
1923 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
1926 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1928 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1929 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1930 return flow_dv_push_vlan_action_resource_register
1931 (dev, &res, dev_flow, error);
1935 * Validate the modify-header actions.
1937 * @param[in] action_flags
1938 * Holds the actions detected until now.
1940 * Pointer to the modify action.
1942 * Pointer to error structure.
1945 * 0 on success, a negative errno value otherwise and rte_errno is set.
1948 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1949 const struct rte_flow_action *action,
1950 struct rte_flow_error *error)
1952 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1953 return rte_flow_error_set(error, EINVAL,
1954 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1955 NULL, "action configuration not set");
1956 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1957 return rte_flow_error_set(error, EINVAL,
1958 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1959 "can't have encap action before"
1965 * Validate the modify-header MAC address actions.
1967 * @param[in] action_flags
1968 * Holds the actions detected until now.
1970 * Pointer to the modify action.
1971 * @param[in] item_flags
1972 * Holds the items detected.
1974 * Pointer to error structure.
1977 * 0 on success, a negative errno value otherwise and rte_errno is set.
1980 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1981 const struct rte_flow_action *action,
1982 const uint64_t item_flags,
1983 struct rte_flow_error *error)
1987 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1989 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1990 return rte_flow_error_set(error, EINVAL,
1991 RTE_FLOW_ERROR_TYPE_ACTION,
1993 "no L2 item in pattern");
1999 * Validate the modify-header IPv4 address actions.
2001 * @param[in] action_flags
2002 * Holds the actions detected until now.
2004 * Pointer to the modify action.
2005 * @param[in] item_flags
2006 * Holds the items detected.
2008 * Pointer to error structure.
2011 * 0 on success, a negative errno value otherwise and rte_errno is set.
2014 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
2015 const struct rte_flow_action *action,
2016 const uint64_t item_flags,
2017 struct rte_flow_error *error)
2021 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2023 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
2024 return rte_flow_error_set(error, EINVAL,
2025 RTE_FLOW_ERROR_TYPE_ACTION,
2027 "no ipv4 item in pattern");
2033 * Validate the modify-header IPv6 address actions.
2035 * @param[in] action_flags
2036 * Holds the actions detected until now.
2038 * Pointer to the modify action.
2039 * @param[in] item_flags
2040 * Holds the items detected.
2042 * Pointer to error structure.
2045 * 0 on success, a negative errno value otherwise and rte_errno is set.
2048 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
2049 const struct rte_flow_action *action,
2050 const uint64_t item_flags,
2051 struct rte_flow_error *error)
2055 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2057 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
2058 return rte_flow_error_set(error, EINVAL,
2059 RTE_FLOW_ERROR_TYPE_ACTION,
2061 "no ipv6 item in pattern");
2067 * Validate the modify-header TP actions.
2069 * @param[in] action_flags
2070 * Holds the actions detected until now.
2072 * Pointer to the modify action.
2073 * @param[in] item_flags
2074 * Holds the items detected.
2076 * Pointer to error structure.
2079 * 0 on success, a negative errno value otherwise and rte_errno is set.
2082 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
2083 const struct rte_flow_action *action,
2084 const uint64_t item_flags,
2085 struct rte_flow_error *error)
2089 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2091 if (!(item_flags & MLX5_FLOW_LAYER_L4))
2092 return rte_flow_error_set(error, EINVAL,
2093 RTE_FLOW_ERROR_TYPE_ACTION,
2094 NULL, "no transport layer "
2101 * Validate the modify-header actions of increment/decrement
2102 * TCP Sequence-number.
2104 * @param[in] action_flags
2105 * Holds the actions detected until now.
2107 * Pointer to the modify action.
2108 * @param[in] item_flags
2109 * Holds the items detected.
2111 * Pointer to error structure.
2114 * 0 on success, a negative errno value otherwise and rte_errno is set.
2117 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
2118 const struct rte_flow_action *action,
2119 const uint64_t item_flags,
2120 struct rte_flow_error *error)
2124 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2126 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2127 return rte_flow_error_set(error, EINVAL,
2128 RTE_FLOW_ERROR_TYPE_ACTION,
2129 NULL, "no TCP item in"
2131 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
2132 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
2133 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
2134 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
2135 return rte_flow_error_set(error, EINVAL,
2136 RTE_FLOW_ERROR_TYPE_ACTION,
2138 "cannot decrease and increase"
2139 " TCP sequence number"
2140 " at the same time");
2146 * Validate the modify-header actions of increment/decrement
2147 * TCP Acknowledgment number.
2149 * @param[in] action_flags
2150 * Holds the actions detected until now.
2152 * Pointer to the modify action.
2153 * @param[in] item_flags
2154 * Holds the items detected.
2156 * Pointer to error structure.
2159 * 0 on success, a negative errno value otherwise and rte_errno is set.
2162 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
2163 const struct rte_flow_action *action,
2164 const uint64_t item_flags,
2165 struct rte_flow_error *error)
2169 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2171 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
2172 return rte_flow_error_set(error, EINVAL,
2173 RTE_FLOW_ERROR_TYPE_ACTION,
2174 NULL, "no TCP item in"
2176 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
2177 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
2178 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
2179 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
2180 return rte_flow_error_set(error, EINVAL,
2181 RTE_FLOW_ERROR_TYPE_ACTION,
2183 "cannot decrease and increase"
2184 " TCP acknowledgment number"
2185 " at the same time");
2191 * Validate the modify-header TTL actions.
2193 * @param[in] action_flags
2194 * Holds the actions detected until now.
2196 * Pointer to the modify action.
2197 * @param[in] item_flags
2198 * Holds the items detected.
2200 * Pointer to error structure.
2203 * 0 on success, a negative errno value otherwise and rte_errno is set.
2206 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
2207 const struct rte_flow_action *action,
2208 const uint64_t item_flags,
2209 struct rte_flow_error *error)
2213 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
2215 if (!(item_flags & MLX5_FLOW_LAYER_L3))
2216 return rte_flow_error_set(error, EINVAL,
2217 RTE_FLOW_ERROR_TYPE_ACTION,
2219 "no IP protocol in pattern");
2225 * Validate jump action.
2228 * Pointer to the jump action.
2229 * @param[in] action_flags
2230 * Holds the actions detected until now.
2232 * The group of the current flow.
2234 * Pointer to error structure.
2237 * 0 on success, a negative errno value otherwise and rte_errno is set.
2240 flow_dv_validate_action_jump(const struct rte_flow_action *action,
2241 uint64_t action_flags,
2243 struct rte_flow_error *error)
2245 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2246 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2247 return rte_flow_error_set(error, EINVAL,
2248 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2249 "can't have 2 fate actions in"
2252 return rte_flow_error_set(error, EINVAL,
2253 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2254 NULL, "action configuration not set");
2255 if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
2256 return rte_flow_error_set(error, EINVAL,
2257 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2258 "target group must be higher then"
2259 " the current flow group");
2264 * Validate the port_id action.
2267 * Pointer to rte_eth_dev structure.
2268 * @param[in] action_flags
2269 * Bit-fields that holds the actions detected until now.
2271 * Port_id RTE action structure.
2273 * Attributes of flow that includes this action.
2275 * Pointer to error structure.
2278 * 0 on success, a negative errno value otherwise and rte_errno is set.
2281 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
2282 uint64_t action_flags,
2283 const struct rte_flow_action *action,
2284 const struct rte_flow_attr *attr,
2285 struct rte_flow_error *error)
2287 const struct rte_flow_action_port_id *port_id;
2289 uint16_t esw_domain_id;
2290 uint16_t act_port_domain_id;
2293 if (!attr->transfer)
2294 return rte_flow_error_set(error, ENOTSUP,
2295 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2297 "port id action is valid in transfer"
2299 if (!action || !action->conf)
2300 return rte_flow_error_set(error, ENOTSUP,
2301 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2303 "port id action parameters must be"
2305 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2306 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2307 return rte_flow_error_set(error, EINVAL,
2308 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2309 "can have only one fate actions in"
2311 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2312 &esw_domain_id, NULL);
2314 return rte_flow_error_set(error, -ret,
2315 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2317 "failed to obtain E-Switch info");
2318 port_id = action->conf;
2319 port = port_id->original ? dev->data->port_id : port_id->id;
2320 ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2322 return rte_flow_error_set
2324 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2325 "failed to obtain E-Switch port id for port");
2326 if (act_port_domain_id != esw_domain_id)
2327 return rte_flow_error_set
2329 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2330 "port does not belong to"
2331 " E-Switch being configured");
2336 * Find existing modify-header resource or create and register a new one.
2338 * @param dev[in, out]
2339 * Pointer to rte_eth_dev structure.
2340 * @param[in, out] resource
2341 * Pointer to modify-header resource.
2342 * @parm[in, out] dev_flow
2343 * Pointer to the dev_flow.
2345 * pointer to error structure.
2348 * 0 on success otherwise -errno and errno is set.
2351 flow_dv_modify_hdr_resource_register
2352 (struct rte_eth_dev *dev,
2353 struct mlx5_flow_dv_modify_hdr_resource *resource,
2354 struct mlx5_flow *dev_flow,
2355 struct rte_flow_error *error)
2357 struct mlx5_priv *priv = dev->data->dev_private;
2358 struct mlx5_ibv_shared *sh = priv->sh;
2359 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2360 struct mlx5dv_dr_domain *ns;
2362 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2363 ns = sh->fdb_domain;
2364 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2369 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2370 /* Lookup a matching resource from cache. */
2371 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2372 if (resource->ft_type == cache_resource->ft_type &&
2373 resource->actions_num == cache_resource->actions_num &&
2374 resource->flags == cache_resource->flags &&
2375 !memcmp((const void *)resource->actions,
2376 (const void *)cache_resource->actions,
2377 (resource->actions_num *
2378 sizeof(resource->actions[0])))) {
2379 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2380 (void *)cache_resource,
2381 rte_atomic32_read(&cache_resource->refcnt));
2382 rte_atomic32_inc(&cache_resource->refcnt);
2383 dev_flow->dv.modify_hdr = cache_resource;
2387 /* Register new modify-header resource. */
2388 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2389 if (!cache_resource)
2390 return rte_flow_error_set(error, ENOMEM,
2391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2392 "cannot allocate resource memory");
2393 *cache_resource = *resource;
2394 cache_resource->verbs_action =
2395 mlx5_glue->dv_create_flow_action_modify_header
2396 (sh->ctx, cache_resource->ft_type,
2397 ns, cache_resource->flags,
2398 cache_resource->actions_num *
2399 sizeof(cache_resource->actions[0]),
2400 (uint64_t *)cache_resource->actions);
2401 if (!cache_resource->verbs_action) {
2402 rte_free(cache_resource);
2403 return rte_flow_error_set(error, ENOMEM,
2404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2405 NULL, "cannot create action");
2407 rte_atomic32_init(&cache_resource->refcnt);
2408 rte_atomic32_inc(&cache_resource->refcnt);
2409 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2410 dev_flow->dv.modify_hdr = cache_resource;
2411 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2412 (void *)cache_resource,
2413 rte_atomic32_read(&cache_resource->refcnt));
2417 #define MLX5_CNT_CONTAINER_RESIZE 64
2420 * Get or create a flow counter.
2423 * Pointer to the Ethernet device structure.
2425 * Indicate if this counter is shared with other flows.
2427 * Counter identifier.
2430 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2432 static struct mlx5_flow_counter *
2433 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
2436 struct mlx5_priv *priv = dev->data->dev_private;
2437 struct mlx5_flow_counter *cnt = NULL;
2438 struct mlx5_devx_obj *dcs = NULL;
2440 if (!priv->config.devx) {
2441 rte_errno = ENOTSUP;
2445 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
2446 if (cnt->shared && cnt->id == id) {
2452 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2455 cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2457 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2461 struct mlx5_flow_counter tmpl = {
2467 tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2469 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
2475 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
2480 * Release a flow counter.
2483 * Pointer to the Ethernet device structure.
2484 * @param[in] counter
2485 * Pointer to the counter handler.
2488 flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
2489 struct mlx5_flow_counter *counter)
2491 struct mlx5_priv *priv = dev->data->dev_private;
2495 if (--counter->ref_cnt == 0) {
2496 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
2497 claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
2503 * Query a devx flow counter.
2506 * Pointer to the Ethernet device structure.
2508 * Pointer to the flow counter.
2510 * The statistics value of packets.
2512 * The statistics value of bytes.
2515 * 0 on success, otherwise a negative errno value and rte_errno is set.
2518 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
2519 struct mlx5_flow_counter *cnt, uint64_t *pkts,
2522 return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
2527 * Get a pool by a counter.
2530 * Pointer to the counter.
2535 static struct mlx5_flow_counter_pool *
2536 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
2539 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
2540 return (struct mlx5_flow_counter_pool *)cnt - 1;
2546 * Get a pool by devx counter ID.
2549 * Pointer to the counter container.
2551 * The counter devx ID.
2554 * The counter pool pointer if exists, NULL otherwise,
2556 static struct mlx5_flow_counter_pool *
2557 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
2559 struct mlx5_flow_counter_pool *pool;
2561 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2562 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
2563 MLX5_COUNTERS_PER_POOL;
2565 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
2572 * Allocate a new memory for the counter values wrapped by all the needed
2576 * Pointer to the Ethernet device structure.
2578 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
2581 * The new memory management pointer on success, otherwise NULL and rte_errno
2584 static struct mlx5_counter_stats_mem_mng *
2585 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
2587 struct mlx5_ibv_shared *sh = ((struct mlx5_priv *)
2588 (dev->data->dev_private))->sh;
2589 struct mlx5_devx_mkey_attr mkey_attr;
2590 struct mlx5_counter_stats_mem_mng *mem_mng;
2591 volatile struct flow_counter_stats *raw_data;
2592 int size = (sizeof(struct flow_counter_stats) *
2593 MLX5_COUNTERS_PER_POOL +
2594 sizeof(struct mlx5_counter_stats_raw)) * raws_n +
2595 sizeof(struct mlx5_counter_stats_mem_mng);
2596 uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
2603 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
2604 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
2605 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
2606 IBV_ACCESS_LOCAL_WRITE);
2607 if (!mem_mng->umem) {
2612 mkey_attr.addr = (uintptr_t)mem;
2613 mkey_attr.size = size;
2614 mkey_attr.umem_id = mem_mng->umem->umem_id;
2615 mkey_attr.pd = sh->pdn;
2616 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
2618 mlx5_glue->devx_umem_dereg(mem_mng->umem);
2623 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
2624 raw_data = (volatile struct flow_counter_stats *)mem;
2625 for (i = 0; i < raws_n; ++i) {
2626 mem_mng->raws[i].mem_mng = mem_mng;
2627 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
2629 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
2634 * Resize a counter container.
2637 * Pointer to the Ethernet device structure.
2639 * Whether the pool is for counter that was allocated by batch command.
2642 * The new container pointer on success, otherwise NULL and rte_errno is set.
2644 static struct mlx5_pools_container *
2645 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
2647 struct mlx5_priv *priv = dev->data->dev_private;
2648 struct mlx5_pools_container *cont =
2649 MLX5_CNT_CONTAINER(priv->sh, batch, 0);
2650 struct mlx5_pools_container *new_cont =
2651 MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
2652 struct mlx5_counter_stats_mem_mng *mem_mng;
2653 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
2654 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
2657 if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
2658 /* The last resize still hasn't detected by the host thread. */
2662 new_cont->pools = rte_calloc(__func__, 1, mem_size, 0);
2663 if (!new_cont->pools) {
2668 memcpy(new_cont->pools, cont->pools, cont->n *
2669 sizeof(struct mlx5_flow_counter_pool *));
2670 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
2671 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
2673 rte_free(new_cont->pools);
2676 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
2677 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
2678 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
2680 new_cont->n = resize;
2681 rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
2682 TAILQ_INIT(&new_cont->pool_list);
2683 TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next);
2684 new_cont->init_mem_mng = mem_mng;
2686 /* Flip the master container. */
2687 priv->sh->cmng.mhi[batch] ^= (uint8_t)1;
2692 * Query a devx flow counter.
2695 * Pointer to the Ethernet device structure.
2697 * Pointer to the flow counter.
2699 * The statistics value of packets.
2701 * The statistics value of bytes.
2704 * 0 on success, otherwise a negative errno value and rte_errno is set.
2707 _flow_dv_query_count(struct rte_eth_dev *dev,
2708 struct mlx5_flow_counter *cnt, uint64_t *pkts,
2711 struct mlx5_priv *priv = dev->data->dev_private;
2712 struct mlx5_flow_counter_pool *pool =
2713 flow_dv_counter_pool_get(cnt);
2714 int offset = cnt - &pool->counters_raw[0];
2716 if (priv->counter_fallback)
2717 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
2719 rte_spinlock_lock(&pool->sl);
2721 * The single counters allocation may allocate smaller ID than the
2722 * current allocated in parallel to the host reading.
2723 * In this case the new counter values must be reported as 0.
2725 if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
2729 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
2730 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
2732 rte_spinlock_unlock(&pool->sl);
2737 * Create and initialize a new counter pool.
2740 * Pointer to the Ethernet device structure.
2742 * The devX counter handle.
2744 * Whether the pool is for counter that was allocated by batch command.
2747 * A new pool pointer on success, NULL otherwise and rte_errno is set.
2749 static struct mlx5_flow_counter_pool *
2750 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
2753 struct mlx5_priv *priv = dev->data->dev_private;
2754 struct mlx5_flow_counter_pool *pool;
2755 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2757 int16_t n_valid = rte_atomic16_read(&cont->n_valid);
2760 if (cont->n == n_valid) {
2761 cont = flow_dv_container_resize(dev, batch);
2765 size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
2766 sizeof(struct mlx5_flow_counter);
2767 pool = rte_calloc(__func__, 1, size, 0);
2772 pool->min_dcs = dcs;
2773 pool->raw = cont->init_mem_mng->raws + n_valid %
2774 MLX5_CNT_CONTAINER_RESIZE;
2775 pool->raw_hw = NULL;
2776 rte_spinlock_init(&pool->sl);
2778 * The generation of the new allocated counters in this pool is 0, 2 in
2779 * the pool generation makes all the counters valid for allocation.
2781 rte_atomic64_set(&pool->query_gen, 0x2);
2782 TAILQ_INIT(&pool->counters);
2783 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
2784 cont->pools[n_valid] = pool;
2785 /* Pool initialization must be updated before host thread access. */
2787 rte_atomic16_add(&cont->n_valid, 1);
2792 * Prepare a new counter and/or a new counter pool.
2795 * Pointer to the Ethernet device structure.
2796 * @param[out] cnt_free
2797 * Where to put the pointer of a new counter.
2799 * Whether the pool is for counter that was allocated by batch command.
2802 * The free counter pool pointer and @p cnt_free is set on success,
2803 * NULL otherwise and rte_errno is set.
2805 static struct mlx5_flow_counter_pool *
2806 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
2807 struct mlx5_flow_counter **cnt_free,
2810 struct mlx5_priv *priv = dev->data->dev_private;
2811 struct mlx5_flow_counter_pool *pool;
2812 struct mlx5_devx_obj *dcs = NULL;
2813 struct mlx5_flow_counter *cnt;
2817 /* bulk_bitmap must be 0 for single counter allocation. */
2818 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
2821 pool = flow_dv_find_pool_by_id
2822 (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id);
2824 pool = flow_dv_pool_create(dev, dcs, batch);
2826 mlx5_devx_cmd_destroy(dcs);
2829 } else if (dcs->id < pool->min_dcs->id) {
2830 rte_atomic64_set(&pool->a64_dcs,
2831 (int64_t)(uintptr_t)dcs);
2833 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
2834 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2839 /* bulk_bitmap is in 128 counters units. */
2840 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
2841 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
2843 rte_errno = ENODATA;
2846 pool = flow_dv_pool_create(dev, dcs, batch);
2848 mlx5_devx_cmd_destroy(dcs);
2851 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2852 cnt = &pool->counters_raw[i];
2854 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
2856 *cnt_free = &pool->counters_raw[0];
2861 * Search for existed shared counter.
2864 * Pointer to the relevant counter pool container.
2866 * The shared counter ID to search.
2869 * NULL if not existed, otherwise pointer to the shared counter.
2871 static struct mlx5_flow_counter *
2872 flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
2875 static struct mlx5_flow_counter *cnt;
2876 struct mlx5_flow_counter_pool *pool;
2879 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2880 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
2881 cnt = &pool->counters_raw[i];
2882 if (cnt->ref_cnt && cnt->shared && cnt->id == id)
2890 * Allocate a flow counter.
2893 * Pointer to the Ethernet device structure.
2895 * Indicate if this counter is shared with other flows.
2897 * Counter identifier.
2899 * Counter flow group.
2902 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2904 static struct mlx5_flow_counter *
2905 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
2908 struct mlx5_priv *priv = dev->data->dev_private;
2909 struct mlx5_flow_counter_pool *pool = NULL;
2910 struct mlx5_flow_counter *cnt_free = NULL;
2912 * Currently group 0 flow counter cannot be assigned to a flow if it is
2913 * not the first one in the batch counter allocation, so it is better
2914 * to allocate counters one by one for these flows in a separate
2916 * A counter can be shared between different groups so need to take
2917 * shared counters from the single container.
2919 uint32_t batch = (group && !shared) ? 1 : 0;
2920 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
2923 if (priv->counter_fallback)
2924 return flow_dv_counter_alloc_fallback(dev, shared, id);
2925 if (!priv->config.devx) {
2926 rte_errno = ENOTSUP;
2930 cnt_free = flow_dv_counter_shared_search(cont, id);
2932 if (cnt_free->ref_cnt + 1 == 0) {
2936 cnt_free->ref_cnt++;
2940 /* Pools which has a free counters are in the start. */
2941 TAILQ_FOREACH(pool, &cont->pool_list, next) {
2943 * The free counter reset values must be updated between the
2944 * counter release to the counter allocation, so, at least one
2945 * query must be done in this time. ensure it by saving the
2946 * query generation in the release time.
2947 * The free list is sorted according to the generation - so if
2948 * the first one is not updated, all the others are not
2951 cnt_free = TAILQ_FIRST(&pool->counters);
2952 if (cnt_free && cnt_free->query_gen + 1 <
2953 rte_atomic64_read(&pool->query_gen))
2958 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
2962 cnt_free->batch = batch;
2963 /* Create a DV counter action only in the first time usage. */
2964 if (!cnt_free->action) {
2966 struct mlx5_devx_obj *dcs;
2969 offset = cnt_free - &pool->counters_raw[0];
2970 dcs = pool->min_dcs;
2973 dcs = cnt_free->dcs;
2975 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
2977 if (!cnt_free->action) {
2982 /* Update the counter reset values. */
2983 if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
2986 cnt_free->shared = shared;
2987 cnt_free->ref_cnt = 1;
2989 if (!priv->sh->cmng.query_thread_on)
2990 /* Start the asynchronous batch query by the host thread. */
2991 mlx5_set_query_alarm(priv->sh);
2992 TAILQ_REMOVE(&pool->counters, cnt_free, next);
2993 if (TAILQ_EMPTY(&pool->counters)) {
2994 /* Move the pool to the end of the container pool list. */
2995 TAILQ_REMOVE(&cont->pool_list, pool, next);
2996 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
3002 * Release a flow counter.
3005 * Pointer to the Ethernet device structure.
3006 * @param[in] counter
3007 * Pointer to the counter handler.
3010 flow_dv_counter_release(struct rte_eth_dev *dev,
3011 struct mlx5_flow_counter *counter)
3013 struct mlx5_priv *priv = dev->data->dev_private;
3017 if (priv->counter_fallback) {
3018 flow_dv_counter_release_fallback(dev, counter);
3021 if (--counter->ref_cnt == 0) {
3022 struct mlx5_flow_counter_pool *pool =
3023 flow_dv_counter_pool_get(counter);
3025 /* Put the counter in the end - the last updated one. */
3026 TAILQ_INSERT_TAIL(&pool->counters, counter, next);
3027 counter->query_gen = rte_atomic64_read(&pool->query_gen);
3032 * Verify the @p attributes will be correctly understood by the NIC and store
3033 * them in the @p flow if everything is correct.
3036 * Pointer to dev struct.
3037 * @param[in] attributes
3038 * Pointer to flow attributes
3040 * Pointer to error structure.
3043 * 0 on success, a negative errno value otherwise and rte_errno is set.
3046 flow_dv_validate_attributes(struct rte_eth_dev *dev,
3047 const struct rte_flow_attr *attributes,
3048 struct rte_flow_error *error)
3050 struct mlx5_priv *priv = dev->data->dev_private;
3051 uint32_t priority_max = priv->config.flow_prio - 1;
3053 #ifndef HAVE_MLX5DV_DR
3054 if (attributes->group)
3055 return rte_flow_error_set(error, ENOTSUP,
3056 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
3058 "groups is not supported");
3060 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
3061 attributes->priority >= priority_max)
3062 return rte_flow_error_set(error, ENOTSUP,
3063 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
3065 "priority out of range");
3066 if (attributes->transfer) {
3067 if (!priv->config.dv_esw_en)
3068 return rte_flow_error_set
3070 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3071 "E-Switch dr is not supported");
3072 if (!(priv->representor || priv->master))
3073 return rte_flow_error_set
3074 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3075 NULL, "E-Switch configuration can only be"
3076 " done by a master or a representor device");
3077 if (attributes->egress)
3078 return rte_flow_error_set
3080 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
3081 "egress is not supported");
3082 if (attributes->group >= MLX5_MAX_TABLES_FDB)
3083 return rte_flow_error_set
3085 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
3086 NULL, "group must be smaller than "
3087 RTE_STR(MLX5_MAX_TABLES_FDB));
3089 if (!(attributes->egress ^ attributes->ingress))
3090 return rte_flow_error_set(error, ENOTSUP,
3091 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
3092 "must specify exactly one of "
3093 "ingress or egress");
3098 * Internal validation function. For validating both actions and items.
3101 * Pointer to the rte_eth_dev structure.
3103 * Pointer to the flow attributes.
3105 * Pointer to the list of items.
3106 * @param[in] actions
3107 * Pointer to the list of actions.
3109 * Pointer to the error structure.
3112 * 0 on success, a negative errno value otherwise and rte_errno is set.
3115 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
3116 const struct rte_flow_item items[],
3117 const struct rte_flow_action actions[],
3118 struct rte_flow_error *error)
3121 uint64_t action_flags = 0;
3122 uint64_t item_flags = 0;
3123 uint64_t last_item = 0;
3124 uint8_t next_protocol = 0xff;
3126 const struct rte_flow_item *gre_item = NULL;
3127 struct rte_flow_item_tcp nic_tcp_mask = {
3130 .src_port = RTE_BE16(UINT16_MAX),
3131 .dst_port = RTE_BE16(UINT16_MAX),
3137 ret = flow_dv_validate_attributes(dev, attr, error);
3140 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3141 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3142 switch (items->type) {
3143 case RTE_FLOW_ITEM_TYPE_VOID:
3145 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3146 ret = flow_dv_validate_item_port_id
3147 (dev, items, attr, item_flags, error);
3150 last_item = MLX5_FLOW_ITEM_PORT_ID;
3152 case RTE_FLOW_ITEM_TYPE_ETH:
3153 ret = mlx5_flow_validate_item_eth(items, item_flags,
3157 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
3158 MLX5_FLOW_LAYER_OUTER_L2;
3160 case RTE_FLOW_ITEM_TYPE_VLAN:
3161 ret = mlx5_flow_validate_item_vlan(items, item_flags,
3165 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
3166 MLX5_FLOW_LAYER_OUTER_VLAN;
3168 case RTE_FLOW_ITEM_TYPE_IPV4:
3169 mlx5_flow_tunnel_ip_check(items, next_protocol,
3170 &item_flags, &tunnel);
3171 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
3175 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3176 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3177 if (items->mask != NULL &&
3178 ((const struct rte_flow_item_ipv4 *)
3179 items->mask)->hdr.next_proto_id) {
3181 ((const struct rte_flow_item_ipv4 *)
3182 (items->spec))->hdr.next_proto_id;
3184 ((const struct rte_flow_item_ipv4 *)
3185 (items->mask))->hdr.next_proto_id;
3187 /* Reset for inner layer. */
3188 next_protocol = 0xff;
3191 case RTE_FLOW_ITEM_TYPE_IPV6:
3192 mlx5_flow_tunnel_ip_check(items, next_protocol,
3193 &item_flags, &tunnel);
3194 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
3198 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3199 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3200 if (items->mask != NULL &&
3201 ((const struct rte_flow_item_ipv6 *)
3202 items->mask)->hdr.proto) {
3204 ((const struct rte_flow_item_ipv6 *)
3205 items->spec)->hdr.proto;
3207 ((const struct rte_flow_item_ipv6 *)
3208 items->mask)->hdr.proto;
3210 /* Reset for inner layer. */
3211 next_protocol = 0xff;
3214 case RTE_FLOW_ITEM_TYPE_TCP:
3215 ret = mlx5_flow_validate_item_tcp
3222 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
3223 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3225 case RTE_FLOW_ITEM_TYPE_UDP:
3226 ret = mlx5_flow_validate_item_udp(items, item_flags,
3231 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
3232 MLX5_FLOW_LAYER_OUTER_L4_UDP;
3234 case RTE_FLOW_ITEM_TYPE_GRE:
3235 ret = mlx5_flow_validate_item_gre(items, item_flags,
3236 next_protocol, error);
3240 last_item = MLX5_FLOW_LAYER_GRE;
3242 case RTE_FLOW_ITEM_TYPE_NVGRE:
3243 ret = mlx5_flow_validate_item_nvgre(items, item_flags,
3248 last_item = MLX5_FLOW_LAYER_NVGRE;
3250 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
3251 ret = mlx5_flow_validate_item_gre_key
3252 (items, item_flags, gre_item, error);
3255 last_item = MLX5_FLOW_LAYER_GRE_KEY;
3257 case RTE_FLOW_ITEM_TYPE_VXLAN:
3258 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
3262 last_item = MLX5_FLOW_LAYER_VXLAN;
3264 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3265 ret = mlx5_flow_validate_item_vxlan_gpe(items,
3270 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
3272 case RTE_FLOW_ITEM_TYPE_MPLS:
3273 ret = mlx5_flow_validate_item_mpls(dev, items,
3278 last_item = MLX5_FLOW_LAYER_MPLS;
3280 case RTE_FLOW_ITEM_TYPE_META:
3281 ret = flow_dv_validate_item_meta(dev, items, attr,
3285 last_item = MLX5_FLOW_ITEM_METADATA;
3287 case RTE_FLOW_ITEM_TYPE_ICMP:
3288 ret = mlx5_flow_validate_item_icmp(items, item_flags,
3293 last_item = MLX5_FLOW_LAYER_ICMP;
3295 case RTE_FLOW_ITEM_TYPE_ICMP6:
3296 ret = mlx5_flow_validate_item_icmp6(items, item_flags,
3301 last_item = MLX5_FLOW_LAYER_ICMP6;
3304 return rte_flow_error_set(error, ENOTSUP,
3305 RTE_FLOW_ERROR_TYPE_ITEM,
3306 NULL, "item not supported");
3308 item_flags |= last_item;
3310 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3311 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
3312 return rte_flow_error_set(error, ENOTSUP,
3313 RTE_FLOW_ERROR_TYPE_ACTION,
3314 actions, "too many actions");
3315 switch (actions->type) {
3316 case RTE_FLOW_ACTION_TYPE_VOID:
3318 case RTE_FLOW_ACTION_TYPE_PORT_ID:
3319 ret = flow_dv_validate_action_port_id(dev,
3326 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
3329 case RTE_FLOW_ACTION_TYPE_FLAG:
3330 ret = mlx5_flow_validate_action_flag(action_flags,
3334 action_flags |= MLX5_FLOW_ACTION_FLAG;
3337 case RTE_FLOW_ACTION_TYPE_MARK:
3338 ret = mlx5_flow_validate_action_mark(actions,
3343 action_flags |= MLX5_FLOW_ACTION_MARK;
3346 case RTE_FLOW_ACTION_TYPE_DROP:
3347 ret = mlx5_flow_validate_action_drop(action_flags,
3351 action_flags |= MLX5_FLOW_ACTION_DROP;
3354 case RTE_FLOW_ACTION_TYPE_QUEUE:
3355 ret = mlx5_flow_validate_action_queue(actions,
3360 action_flags |= MLX5_FLOW_ACTION_QUEUE;
3363 case RTE_FLOW_ACTION_TYPE_RSS:
3364 ret = mlx5_flow_validate_action_rss(actions,
3370 action_flags |= MLX5_FLOW_ACTION_RSS;
3373 case RTE_FLOW_ACTION_TYPE_COUNT:
3374 ret = flow_dv_validate_action_count(dev, error);
3377 action_flags |= MLX5_FLOW_ACTION_COUNT;
3380 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3381 if (flow_dv_validate_action_pop_vlan(dev,
3387 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
3390 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3391 ret = flow_dv_validate_action_push_vlan(action_flags,
3396 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
3399 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3400 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3401 ret = flow_dv_validate_action_l2_encap(action_flags,
3406 action_flags |= actions->type ==
3407 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
3408 MLX5_FLOW_ACTION_VXLAN_ENCAP :
3409 MLX5_FLOW_ACTION_NVGRE_ENCAP;
3412 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3413 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3414 ret = flow_dv_validate_action_l2_decap(action_flags,
3418 action_flags |= actions->type ==
3419 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
3420 MLX5_FLOW_ACTION_VXLAN_DECAP :
3421 MLX5_FLOW_ACTION_NVGRE_DECAP;
3424 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3425 ret = flow_dv_validate_action_raw_encap(action_flags,
3430 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
3433 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3434 ret = flow_dv_validate_action_raw_decap(action_flags,
3439 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
3442 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3443 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3444 ret = flow_dv_validate_action_modify_mac(action_flags,
3450 /* Count all modify-header actions as one action. */
3451 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3453 action_flags |= actions->type ==
3454 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
3455 MLX5_FLOW_ACTION_SET_MAC_SRC :
3456 MLX5_FLOW_ACTION_SET_MAC_DST;
3459 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3460 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3461 ret = flow_dv_validate_action_modify_ipv4(action_flags,
3467 /* Count all modify-header actions as one action. */
3468 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3470 action_flags |= actions->type ==
3471 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
3472 MLX5_FLOW_ACTION_SET_IPV4_SRC :
3473 MLX5_FLOW_ACTION_SET_IPV4_DST;
3475 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3476 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3477 ret = flow_dv_validate_action_modify_ipv6(action_flags,
3483 /* Count all modify-header actions as one action. */
3484 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3486 action_flags |= actions->type ==
3487 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
3488 MLX5_FLOW_ACTION_SET_IPV6_SRC :
3489 MLX5_FLOW_ACTION_SET_IPV6_DST;
3491 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3492 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3493 ret = flow_dv_validate_action_modify_tp(action_flags,
3499 /* Count all modify-header actions as one action. */
3500 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3502 action_flags |= actions->type ==
3503 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
3504 MLX5_FLOW_ACTION_SET_TP_SRC :
3505 MLX5_FLOW_ACTION_SET_TP_DST;
3507 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3508 case RTE_FLOW_ACTION_TYPE_SET_TTL:
3509 ret = flow_dv_validate_action_modify_ttl(action_flags,
3515 /* Count all modify-header actions as one action. */
3516 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3518 action_flags |= actions->type ==
3519 RTE_FLOW_ACTION_TYPE_SET_TTL ?
3520 MLX5_FLOW_ACTION_SET_TTL :
3521 MLX5_FLOW_ACTION_DEC_TTL;
3523 case RTE_FLOW_ACTION_TYPE_JUMP:
3524 ret = flow_dv_validate_action_jump(actions,
3526 attr->group, error);
3530 action_flags |= MLX5_FLOW_ACTION_JUMP;
3532 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3533 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3534 ret = flow_dv_validate_action_modify_tcp_seq
3541 /* Count all modify-header actions as one action. */
3542 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3544 action_flags |= actions->type ==
3545 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
3546 MLX5_FLOW_ACTION_INC_TCP_SEQ :
3547 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
3549 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3550 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3551 ret = flow_dv_validate_action_modify_tcp_ack
3558 /* Count all modify-header actions as one action. */
3559 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
3561 action_flags |= actions->type ==
3562 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
3563 MLX5_FLOW_ACTION_INC_TCP_ACK :
3564 MLX5_FLOW_ACTION_DEC_TCP_ACK;
3567 return rte_flow_error_set(error, ENOTSUP,
3568 RTE_FLOW_ERROR_TYPE_ACTION,
3570 "action not supported");
3573 if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3574 (action_flags & MLX5_FLOW_VLAN_ACTIONS))
3575 return rte_flow_error_set(error, ENOTSUP,
3576 RTE_FLOW_ERROR_TYPE_ACTION,
3578 "can't have vxlan and vlan"
3579 " actions in the same rule");
3580 /* Eswitch has few restrictions on using items and actions */
3581 if (attr->transfer) {
3582 if (action_flags & MLX5_FLOW_ACTION_FLAG)
3583 return rte_flow_error_set(error, ENOTSUP,
3584 RTE_FLOW_ERROR_TYPE_ACTION,
3586 "unsupported action FLAG");
3587 if (action_flags & MLX5_FLOW_ACTION_MARK)
3588 return rte_flow_error_set(error, ENOTSUP,
3589 RTE_FLOW_ERROR_TYPE_ACTION,
3591 "unsupported action MARK");
3592 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
3593 return rte_flow_error_set(error, ENOTSUP,
3594 RTE_FLOW_ERROR_TYPE_ACTION,
3596 "unsupported action QUEUE");
3597 if (action_flags & MLX5_FLOW_ACTION_RSS)
3598 return rte_flow_error_set(error, ENOTSUP,
3599 RTE_FLOW_ERROR_TYPE_ACTION,
3601 "unsupported action RSS");
3602 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3603 return rte_flow_error_set(error, EINVAL,
3604 RTE_FLOW_ERROR_TYPE_ACTION,
3606 "no fate action is found");
3608 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
3609 return rte_flow_error_set(error, EINVAL,
3610 RTE_FLOW_ERROR_TYPE_ACTION,
3612 "no fate action is found");
3618 * Internal preparation function. Allocates the DV flow size,
3619 * this size is constant.
3622 * Pointer to the flow attributes.
3624 * Pointer to the list of items.
3625 * @param[in] actions
3626 * Pointer to the list of actions.
3628 * Pointer to the error structure.
3631 * Pointer to mlx5_flow object on success,
3632 * otherwise NULL and rte_errno is set.
3634 static struct mlx5_flow *
3635 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
3636 const struct rte_flow_item items[] __rte_unused,
3637 const struct rte_flow_action actions[] __rte_unused,
3638 struct rte_flow_error *error)
3640 uint32_t size = sizeof(struct mlx5_flow);
3641 struct mlx5_flow *flow;
3643 flow = rte_calloc(__func__, 1, size, 0);
3645 rte_flow_error_set(error, ENOMEM,
3646 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3647 "not enough memory to create flow");
3650 flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
3656 * Sanity check for match mask and value. Similar to check_valid_spec() in
3657 * kernel driver. If unmasked bit is present in value, it returns failure.
3660 * pointer to match mask buffer.
3661 * @param match_value
3662 * pointer to match value buffer.
3665 * 0 if valid, -EINVAL otherwise.
3668 flow_dv_check_valid_spec(void *match_mask, void *match_value)
3670 uint8_t *m = match_mask;
3671 uint8_t *v = match_value;
3674 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
3677 "match_value differs from match_criteria"
3678 " %p[%u] != %p[%u]",
3679 match_value, i, match_mask, i);
3688 * Add Ethernet item to matcher and to the value.
3690 * @param[in, out] matcher
3692 * @param[in, out] key
3693 * Flow matcher value.
3695 * Flow pattern to translate.
3697 * Item is inner pattern.
3700 flow_dv_translate_item_eth(void *matcher, void *key,
3701 const struct rte_flow_item *item, int inner)
3703 const struct rte_flow_item_eth *eth_m = item->mask;
3704 const struct rte_flow_item_eth *eth_v = item->spec;
3705 const struct rte_flow_item_eth nic_mask = {
3706 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3707 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
3708 .type = RTE_BE16(0xffff),
3720 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3722 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3724 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3726 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3728 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
3729 ð_m->dst, sizeof(eth_m->dst));
3730 /* The value must be in the range of the mask. */
3731 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
3732 for (i = 0; i < sizeof(eth_m->dst); ++i)
3733 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
3734 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
3735 ð_m->src, sizeof(eth_m->src));
3736 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
3737 /* The value must be in the range of the mask. */
3738 for (i = 0; i < sizeof(eth_m->dst); ++i)
3739 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
3740 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3741 rte_be_to_cpu_16(eth_m->type));
3742 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
3743 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
3747 * Add VLAN item to matcher and to the value.
3749 * @param[in, out] dev_flow
3751 * @param[in, out] matcher
3753 * @param[in, out] key
3754 * Flow matcher value.
3756 * Flow pattern to translate.
3758 * Item is inner pattern.
3761 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
3762 void *matcher, void *key,
3763 const struct rte_flow_item *item,
3766 const struct rte_flow_item_vlan *vlan_m = item->mask;
3767 const struct rte_flow_item_vlan *vlan_v = item->spec;
3776 vlan_m = &rte_flow_item_vlan_mask;
3778 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3780 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3782 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3784 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3786 * This is workaround, masks are not supported,
3787 * and pre-validated.
3789 dev_flow->dv.vf_vlan.tag =
3790 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
3792 tci_m = rte_be_to_cpu_16(vlan_m->tci);
3793 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
3794 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
3795 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
3796 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
3797 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
3798 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
3799 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
3800 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
3801 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
3802 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
3803 rte_be_to_cpu_16(vlan_m->inner_type));
3804 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
3805 rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
3809 * Add IPV4 item to matcher and to the value.
3811 * @param[in, out] matcher
3813 * @param[in, out] key
3814 * Flow matcher value.
3816 * Flow pattern to translate.
3818 * Item is inner pattern.
3820 * The group to insert the rule.
3823 flow_dv_translate_item_ipv4(void *matcher, void *key,
3824 const struct rte_flow_item *item,
3825 int inner, uint32_t group)
3827 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
3828 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
3829 const struct rte_flow_item_ipv4 nic_mask = {
3831 .src_addr = RTE_BE32(0xffffffff),
3832 .dst_addr = RTE_BE32(0xffffffff),
3833 .type_of_service = 0xff,
3834 .next_proto_id = 0xff,
3844 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3846 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3848 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3850 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3853 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3855 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
3856 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
3861 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3862 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3863 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3864 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
3865 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
3866 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
3867 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3868 src_ipv4_src_ipv6.ipv4_layout.ipv4);
3869 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3870 src_ipv4_src_ipv6.ipv4_layout.ipv4);
3871 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
3872 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
3873 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
3874 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
3875 ipv4_m->hdr.type_of_service);
3876 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
3877 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
3878 ipv4_m->hdr.type_of_service >> 2);
3879 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
3880 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3881 ipv4_m->hdr.next_proto_id);
3882 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3883 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3887 * Add IPV6 item to matcher and to the value.
3889 * @param[in, out] matcher
3891 * @param[in, out] key
3892 * Flow matcher value.
3894 * Flow pattern to translate.
3896 * Item is inner pattern.
3898 * The group to insert the rule.
3901 flow_dv_translate_item_ipv6(void *matcher, void *key,
3902 const struct rte_flow_item *item,
3903 int inner, uint32_t group)
3905 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
3906 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
3907 const struct rte_flow_item_ipv6 nic_mask = {
3910 "\xff\xff\xff\xff\xff\xff\xff\xff"
3911 "\xff\xff\xff\xff\xff\xff\xff\xff",
3913 "\xff\xff\xff\xff\xff\xff\xff\xff"
3914 "\xff\xff\xff\xff\xff\xff\xff\xff",
3915 .vtc_flow = RTE_BE32(0xffffffff),
3922 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3923 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3932 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3934 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3936 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3938 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3941 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3943 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
3944 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
3949 size = sizeof(ipv6_m->hdr.dst_addr);
3950 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3951 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3952 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3953 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3954 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
3955 for (i = 0; i < size; ++i)
3956 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
3957 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3958 src_ipv4_src_ipv6.ipv6_layout.ipv6);
3959 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3960 src_ipv4_src_ipv6.ipv6_layout.ipv6);
3961 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
3962 for (i = 0; i < size; ++i)
3963 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
3965 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
3966 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
3967 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
3968 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
3969 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
3970 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
3973 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
3975 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
3978 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
3980 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
3984 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3986 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3987 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
3991 * Add TCP item to matcher and to the value.
3993 * @param[in, out] matcher
3995 * @param[in, out] key
3996 * Flow matcher value.
3998 * Flow pattern to translate.
4000 * Item is inner pattern.
4003 flow_dv_translate_item_tcp(void *matcher, void *key,
4004 const struct rte_flow_item *item,
4007 const struct rte_flow_item_tcp *tcp_m = item->mask;
4008 const struct rte_flow_item_tcp *tcp_v = item->spec;
4013 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4015 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4017 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4019 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4021 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4022 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
4026 tcp_m = &rte_flow_item_tcp_mask;
4027 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
4028 rte_be_to_cpu_16(tcp_m->hdr.src_port));
4029 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
4030 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
4031 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
4032 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
4033 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
4034 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
4035 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
4036 tcp_m->hdr.tcp_flags);
4037 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
4038 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
4042 * Add UDP item to matcher and to the value.
4044 * @param[in, out] matcher
4046 * @param[in, out] key
4047 * Flow matcher value.
4049 * Flow pattern to translate.
4051 * Item is inner pattern.
4054 flow_dv_translate_item_udp(void *matcher, void *key,
4055 const struct rte_flow_item *item,
4058 const struct rte_flow_item_udp *udp_m = item->mask;
4059 const struct rte_flow_item_udp *udp_v = item->spec;
4064 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4066 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4068 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4070 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4072 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4073 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
4077 udp_m = &rte_flow_item_udp_mask;
4078 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
4079 rte_be_to_cpu_16(udp_m->hdr.src_port));
4080 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
4081 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
4082 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
4083 rte_be_to_cpu_16(udp_m->hdr.dst_port));
4084 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4085 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
4089 * Add GRE optional Key item to matcher and to the value.
4091 * @param[in, out] matcher
4093 * @param[in, out] key
4094 * Flow matcher value.
4096 * Flow pattern to translate.
4098 * Item is inner pattern.
4101 flow_dv_translate_item_gre_key(void *matcher, void *key,
4102 const struct rte_flow_item *item)
4104 const rte_be32_t *key_m = item->mask;
4105 const rte_be32_t *key_v = item->spec;
4106 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4107 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4108 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
4113 key_m = &gre_key_default_mask;
4114 /* GRE K bit must be on and should already be validated */
4115 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
4116 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
4117 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
4118 rte_be_to_cpu_32(*key_m) >> 8);
4119 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
4120 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
4121 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
4122 rte_be_to_cpu_32(*key_m) & 0xFF);
4123 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
4124 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
4128 * Add GRE item to matcher and to the value.
4130 * @param[in, out] matcher
4132 * @param[in, out] key
4133 * Flow matcher value.
4135 * Flow pattern to translate.
4137 * Item is inner pattern.
4140 flow_dv_translate_item_gre(void *matcher, void *key,
4141 const struct rte_flow_item *item,
4144 const struct rte_flow_item_gre *gre_m = item->mask;
4145 const struct rte_flow_item_gre *gre_v = item->spec;
4148 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4149 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4156 uint16_t s_present:1;
4157 uint16_t k_present:1;
4158 uint16_t rsvd_bit1:1;
4159 uint16_t c_present:1;
4163 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
4166 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4168 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4170 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4172 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4174 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4175 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
4179 gre_m = &rte_flow_item_gre_mask;
4180 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
4181 rte_be_to_cpu_16(gre_m->protocol));
4182 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4183 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
4184 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
4185 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
4186 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
4187 gre_crks_rsvd0_ver_m.c_present);
4188 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
4189 gre_crks_rsvd0_ver_v.c_present &
4190 gre_crks_rsvd0_ver_m.c_present);
4191 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
4192 gre_crks_rsvd0_ver_m.k_present);
4193 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
4194 gre_crks_rsvd0_ver_v.k_present &
4195 gre_crks_rsvd0_ver_m.k_present);
4196 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
4197 gre_crks_rsvd0_ver_m.s_present);
4198 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
4199 gre_crks_rsvd0_ver_v.s_present &
4200 gre_crks_rsvd0_ver_m.s_present);
4204 * Add NVGRE item to matcher and to the value.
4206 * @param[in, out] matcher
4208 * @param[in, out] key
4209 * Flow matcher value.
4211 * Flow pattern to translate.
4213 * Item is inner pattern.
4216 flow_dv_translate_item_nvgre(void *matcher, void *key,
4217 const struct rte_flow_item *item,
4220 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
4221 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
4222 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4223 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4224 const char *tni_flow_id_m = (const char *)nvgre_m->tni;
4225 const char *tni_flow_id_v = (const char *)nvgre_v->tni;
4231 /* For NVGRE, GRE header fields must be set with defined values. */
4232 const struct rte_flow_item_gre gre_spec = {
4233 .c_rsvd0_ver = RTE_BE16(0x2000),
4234 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
4236 const struct rte_flow_item_gre gre_mask = {
4237 .c_rsvd0_ver = RTE_BE16(0xB000),
4238 .protocol = RTE_BE16(UINT16_MAX),
4240 const struct rte_flow_item gre_item = {
4245 flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
4249 nvgre_m = &rte_flow_item_nvgre_mask;
4250 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
4251 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
4252 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
4253 memcpy(gre_key_m, tni_flow_id_m, size);
4254 for (i = 0; i < size; ++i)
4255 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
4259 * Add VXLAN item to matcher and to the value.
4261 * @param[in, out] matcher
4263 * @param[in, out] key
4264 * Flow matcher value.
4266 * Flow pattern to translate.
4268 * Item is inner pattern.
4271 flow_dv_translate_item_vxlan(void *matcher, void *key,
4272 const struct rte_flow_item *item,
4275 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
4276 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
4279 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4280 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4288 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4290 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4292 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4294 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4296 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
4297 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
4298 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
4299 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
4300 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
4305 vxlan_m = &rte_flow_item_vxlan_mask;
4306 size = sizeof(vxlan_m->vni);
4307 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
4308 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
4309 memcpy(vni_m, vxlan_m->vni, size);
4310 for (i = 0; i < size; ++i)
4311 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
4315 * Add MPLS item to matcher and to the value.
4317 * @param[in, out] matcher
4319 * @param[in, out] key
4320 * Flow matcher value.
4322 * Flow pattern to translate.
4323 * @param[in] prev_layer
4324 * The protocol layer indicated in previous item.
4326 * Item is inner pattern.
4329 flow_dv_translate_item_mpls(void *matcher, void *key,
4330 const struct rte_flow_item *item,
4331 uint64_t prev_layer,
4334 const uint32_t *in_mpls_m = item->mask;
4335 const uint32_t *in_mpls_v = item->spec;
4336 uint32_t *out_mpls_m = 0;
4337 uint32_t *out_mpls_v = 0;
4338 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4339 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4340 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
4342 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4343 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
4344 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4346 switch (prev_layer) {
4347 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4348 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
4349 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
4350 MLX5_UDP_PORT_MPLS);
4352 case MLX5_FLOW_LAYER_GRE:
4353 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
4354 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
4355 RTE_ETHER_TYPE_MPLS);
4358 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
4359 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
4366 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
4367 switch (prev_layer) {
4368 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
4370 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4371 outer_first_mpls_over_udp);
4373 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4374 outer_first_mpls_over_udp);
4376 case MLX5_FLOW_LAYER_GRE:
4378 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
4379 outer_first_mpls_over_gre);
4381 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
4382 outer_first_mpls_over_gre);
4385 /* Inner MPLS not over GRE is not supported. */
4388 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4392 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
4398 if (out_mpls_m && out_mpls_v) {
4399 *out_mpls_m = *in_mpls_m;
4400 *out_mpls_v = *in_mpls_v & *in_mpls_m;
4405 * Add META item to matcher
4407 * @param[in, out] matcher
4409 * @param[in, out] key
4410 * Flow matcher value.
4412 * Flow pattern to translate.
4414 * Item is inner pattern.
4417 flow_dv_translate_item_meta(void *matcher, void *key,
4418 const struct rte_flow_item *item)
4420 const struct rte_flow_item_meta *meta_m;
4421 const struct rte_flow_item_meta *meta_v;
4423 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
4425 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
4427 meta_m = (const void *)item->mask;
4429 meta_m = &rte_flow_item_meta_mask;
4430 meta_v = (const void *)item->spec;
4432 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
4433 rte_be_to_cpu_32(meta_m->data));
4434 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
4435 rte_be_to_cpu_32(meta_v->data & meta_m->data));
4440 * Add source vport match to the specified matcher.
4442 * @param[in, out] matcher
4444 * @param[in, out] key
4445 * Flow matcher value.
4447 * Source vport value to match
4452 flow_dv_translate_item_source_vport(void *matcher, void *key,
4453 int16_t port, uint16_t mask)
4455 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
4456 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
4458 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
4459 MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
4463 * Translate port-id item to eswitch match on port-id.
4466 * The devich to configure through.
4467 * @param[in, out] matcher
4469 * @param[in, out] key
4470 * Flow matcher value.
4472 * Flow pattern to translate.
4475 * 0 on success, a negative errno value otherwise.
4478 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
4479 void *key, const struct rte_flow_item *item)
4481 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
4482 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
4483 uint16_t mask, val, id;
4486 mask = pid_m ? pid_m->id : 0xffff;
4487 id = pid_v ? pid_v->id : dev->data->port_id;
4488 ret = mlx5_port_to_eswitch_info(id, NULL, &val);
4491 flow_dv_translate_item_source_vport(matcher, key, val, mask);
4496 * Add ICMP6 item to matcher and to the value.
4498 * @param[in, out] matcher
4500 * @param[in, out] key
4501 * Flow matcher value.
4503 * Flow pattern to translate.
4505 * Item is inner pattern.
4508 flow_dv_translate_item_icmp6(void *matcher, void *key,
4509 const struct rte_flow_item *item,
4512 const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
4513 const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
4516 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4518 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4520 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4522 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4524 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4526 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4528 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4529 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
4533 icmp6_m = &rte_flow_item_icmp6_mask;
4534 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
4535 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
4536 icmp6_v->type & icmp6_m->type);
4537 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
4538 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
4539 icmp6_v->code & icmp6_m->code);
4543 * Add ICMP item to matcher and to the value.
4545 * @param[in, out] matcher
4547 * @param[in, out] key
4548 * Flow matcher value.
4550 * Flow pattern to translate.
4552 * Item is inner pattern.
4555 flow_dv_translate_item_icmp(void *matcher, void *key,
4556 const struct rte_flow_item *item,
4559 const struct rte_flow_item_icmp *icmp_m = item->mask;
4560 const struct rte_flow_item_icmp *icmp_v = item->spec;
4563 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
4565 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
4567 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4569 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
4571 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
4573 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
4575 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
4576 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
4580 icmp_m = &rte_flow_item_icmp_mask;
4581 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
4582 icmp_m->hdr.icmp_type);
4583 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
4584 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
4585 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
4586 icmp_m->hdr.icmp_code);
4587 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
4588 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
4591 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
4593 #define HEADER_IS_ZERO(match_criteria, headers) \
4594 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
4595 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
4598 * Calculate flow matcher enable bitmap.
4600 * @param match_criteria
4601 * Pointer to flow matcher criteria.
4604 * Bitmap of enabled fields.
4607 flow_dv_matcher_enable(uint32_t *match_criteria)
4609 uint8_t match_criteria_enable;
4611 match_criteria_enable =
4612 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
4613 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
4614 match_criteria_enable |=
4615 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
4616 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
4617 match_criteria_enable |=
4618 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
4619 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
4620 match_criteria_enable |=
4621 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
4622 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
4623 match_criteria_enable |=
4624 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
4625 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
4626 return match_criteria_enable;
4633 * @param dev[in, out]
4634 * Pointer to rte_eth_dev structure.
4635 * @param[in] table_id
4638 * Direction of the table.
4639 * @param[in] transfer
4640 * E-Switch or NIC flow.
4642 * pointer to error structure.
4645 * Returns tables resource based on the index, NULL in case of failed.
4647 static struct mlx5_flow_tbl_resource *
4648 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
4649 uint32_t table_id, uint8_t egress,
4651 struct rte_flow_error *error)
4653 struct mlx5_priv *priv = dev->data->dev_private;
4654 struct mlx5_ibv_shared *sh = priv->sh;
4655 struct mlx5_flow_tbl_resource *tbl;
4657 #ifdef HAVE_MLX5DV_DR
4659 tbl = &sh->fdb_tbl[table_id];
4661 tbl->obj = mlx5_glue->dr_create_flow_tbl
4662 (sh->fdb_domain, table_id);
4663 } else if (egress) {
4664 tbl = &sh->tx_tbl[table_id];
4666 tbl->obj = mlx5_glue->dr_create_flow_tbl
4667 (sh->tx_domain, table_id);
4669 tbl = &sh->rx_tbl[table_id];
4671 tbl->obj = mlx5_glue->dr_create_flow_tbl
4672 (sh->rx_domain, table_id);
4675 rte_flow_error_set(error, ENOMEM,
4676 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4677 NULL, "cannot create table");
4680 rte_atomic32_inc(&tbl->refcnt);
4686 return &sh->fdb_tbl[table_id];
4688 return &sh->tx_tbl[table_id];
4690 return &sh->rx_tbl[table_id];
4695 * Release a flow table.
4698 * Table resource to be released.
4701 * Returns 0 if table was released, else return 1;
4704 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
4708 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
4709 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
4717 * Register the flow matcher.
4719 * @param dev[in, out]
4720 * Pointer to rte_eth_dev structure.
4721 * @param[in, out] matcher
4722 * Pointer to flow matcher.
4723 * @parm[in, out] dev_flow
4724 * Pointer to the dev_flow.
4726 * pointer to error structure.
4729 * 0 on success otherwise -errno and errno is set.
4732 flow_dv_matcher_register(struct rte_eth_dev *dev,
4733 struct mlx5_flow_dv_matcher *matcher,
4734 struct mlx5_flow *dev_flow,
4735 struct rte_flow_error *error)
4737 struct mlx5_priv *priv = dev->data->dev_private;
4738 struct mlx5_ibv_shared *sh = priv->sh;
4739 struct mlx5_flow_dv_matcher *cache_matcher;
4740 struct mlx5dv_flow_matcher_attr dv_attr = {
4741 .type = IBV_FLOW_ATTR_NORMAL,
4742 .match_mask = (void *)&matcher->mask,
4744 struct mlx5_flow_tbl_resource *tbl = NULL;
4746 /* Lookup from cache. */
4747 LIST_FOREACH(cache_matcher, &sh->matchers, next) {
4748 if (matcher->crc == cache_matcher->crc &&
4749 matcher->priority == cache_matcher->priority &&
4750 matcher->egress == cache_matcher->egress &&
4751 matcher->group == cache_matcher->group &&
4752 matcher->transfer == cache_matcher->transfer &&
4753 !memcmp((const void *)matcher->mask.buf,
4754 (const void *)cache_matcher->mask.buf,
4755 cache_matcher->mask.size)) {
4757 "priority %hd use %s matcher %p: refcnt %d++",
4758 cache_matcher->priority,
4759 cache_matcher->egress ? "tx" : "rx",
4760 (void *)cache_matcher,
4761 rte_atomic32_read(&cache_matcher->refcnt));
4762 rte_atomic32_inc(&cache_matcher->refcnt);
4763 dev_flow->dv.matcher = cache_matcher;
4767 /* Register new matcher. */
4768 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
4770 return rte_flow_error_set(error, ENOMEM,
4771 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4772 "cannot allocate matcher memory");
4773 tbl = flow_dv_tbl_resource_get(dev, matcher->group,
4774 matcher->egress, matcher->transfer,
4777 rte_free(cache_matcher);
4778 return rte_flow_error_set(error, ENOMEM,
4779 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4780 NULL, "cannot create table");
4782 *cache_matcher = *matcher;
4783 dv_attr.match_criteria_enable =
4784 flow_dv_matcher_enable(cache_matcher->mask.buf);
4785 dv_attr.priority = matcher->priority;
4786 if (matcher->egress)
4787 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
4788 cache_matcher->matcher_object =
4789 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
4790 if (!cache_matcher->matcher_object) {
4791 rte_free(cache_matcher);
4792 #ifdef HAVE_MLX5DV_DR
4793 flow_dv_tbl_resource_release(tbl);
4795 return rte_flow_error_set(error, ENOMEM,
4796 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4797 NULL, "cannot create matcher");
4799 rte_atomic32_inc(&cache_matcher->refcnt);
4800 LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
4801 dev_flow->dv.matcher = cache_matcher;
4802 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
4803 cache_matcher->priority,
4804 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
4805 rte_atomic32_read(&cache_matcher->refcnt));
4806 rte_atomic32_inc(&tbl->refcnt);
4811 * Find existing tag resource or create and register a new one.
4813 * @param dev[in, out]
4814 * Pointer to rte_eth_dev structure.
4815 * @param[in, out] resource
4816 * Pointer to tag resource.
4817 * @parm[in, out] dev_flow
4818 * Pointer to the dev_flow.
4820 * pointer to error structure.
4823 * 0 on success otherwise -errno and errno is set.
4826 flow_dv_tag_resource_register
4827 (struct rte_eth_dev *dev,
4828 struct mlx5_flow_dv_tag_resource *resource,
4829 struct mlx5_flow *dev_flow,
4830 struct rte_flow_error *error)
4832 struct mlx5_priv *priv = dev->data->dev_private;
4833 struct mlx5_ibv_shared *sh = priv->sh;
4834 struct mlx5_flow_dv_tag_resource *cache_resource;
4836 /* Lookup a matching resource from cache. */
4837 LIST_FOREACH(cache_resource, &sh->tags, next) {
4838 if (resource->tag == cache_resource->tag) {
4839 DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
4840 (void *)cache_resource,
4841 rte_atomic32_read(&cache_resource->refcnt));
4842 rte_atomic32_inc(&cache_resource->refcnt);
4843 dev_flow->flow->tag_resource = cache_resource;
4847 /* Register new resource. */
4848 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
4849 if (!cache_resource)
4850 return rte_flow_error_set(error, ENOMEM,
4851 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4852 "cannot allocate resource memory");
4853 *cache_resource = *resource;
4854 cache_resource->action = mlx5_glue->dv_create_flow_action_tag
4856 if (!cache_resource->action) {
4857 rte_free(cache_resource);
4858 return rte_flow_error_set(error, ENOMEM,
4859 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4860 NULL, "cannot create action");
4862 rte_atomic32_init(&cache_resource->refcnt);
4863 rte_atomic32_inc(&cache_resource->refcnt);
4864 LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
4865 dev_flow->flow->tag_resource = cache_resource;
4866 DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
4867 (void *)cache_resource,
4868 rte_atomic32_read(&cache_resource->refcnt));
4876 * Pointer to Ethernet device.
4878 * Pointer to mlx5_flow.
4881 * 1 while a reference on it exists, 0 when freed.
4884 flow_dv_tag_release(struct rte_eth_dev *dev,
4885 struct mlx5_flow_dv_tag_resource *tag)
4888 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
4889 dev->data->port_id, (void *)tag,
4890 rte_atomic32_read(&tag->refcnt));
4891 if (rte_atomic32_dec_and_test(&tag->refcnt)) {
4892 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
4893 LIST_REMOVE(tag, next);
4894 DRV_LOG(DEBUG, "port %u tag %p: removed",
4895 dev->data->port_id, (void *)tag);
4903 * Translate port ID action to vport.
4906 * Pointer to rte_eth_dev structure.
4908 * Pointer to the port ID action.
4909 * @param[out] dst_port_id
4910 * The target port ID.
4912 * Pointer to the error structure.
4915 * 0 on success, a negative errno value otherwise and rte_errno is set.
4918 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
4919 const struct rte_flow_action *action,
4920 uint32_t *dst_port_id,
4921 struct rte_flow_error *error)
4926 const struct rte_flow_action_port_id *conf =
4927 (const struct rte_flow_action_port_id *)action->conf;
4929 port = conf->original ? dev->data->port_id : conf->id;
4930 ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
4932 return rte_flow_error_set(error, -ret,
4933 RTE_FLOW_ERROR_TYPE_ACTION,
4935 "No eswitch info was found for port");
4936 *dst_port_id = port_id;
4941 * Fill the flow with DV spec.
4944 * Pointer to rte_eth_dev structure.
4945 * @param[in, out] dev_flow
4946 * Pointer to the sub flow.
4948 * Pointer to the flow attributes.
4950 * Pointer to the list of items.
4951 * @param[in] actions
4952 * Pointer to the list of actions.
4954 * Pointer to the error structure.
4957 * 0 on success, a negative errno value otherwise and rte_errno is set.
4960 flow_dv_translate(struct rte_eth_dev *dev,
4961 struct mlx5_flow *dev_flow,
4962 const struct rte_flow_attr *attr,
4963 const struct rte_flow_item items[],
4964 const struct rte_flow_action actions[],
4965 struct rte_flow_error *error)
4967 struct mlx5_priv *priv = dev->data->dev_private;
4968 struct rte_flow *flow = dev_flow->flow;
4969 uint64_t item_flags = 0;
4970 uint64_t last_item = 0;
4971 uint64_t action_flags = 0;
4972 uint64_t priority = attr->priority;
4973 struct mlx5_flow_dv_matcher matcher = {
4975 .size = sizeof(matcher.mask.buf),
4979 bool actions_end = false;
4980 struct mlx5_flow_dv_modify_hdr_resource res = {
4981 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4982 MLX5DV_FLOW_TABLE_TYPE_NIC_RX
4984 union flow_dv_attr flow_attr = { .attr = 0 };
4985 struct mlx5_flow_dv_tag_resource tag_resource;
4986 uint32_t modify_action_position = UINT32_MAX;
4987 void *match_mask = matcher.mask.buf;
4988 void *match_value = dev_flow->dv.value.buf;
4989 uint8_t next_protocol = 0xff;
4990 struct rte_vlan_hdr vlan = { 0 };
4991 bool vlan_inherited = false;
4993 flow->group = attr->group;
4995 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4996 if (priority == MLX5_FLOW_PRIO_RSVD)
4997 priority = priv->config.flow_prio - 1;
4998 for (; !actions_end ; actions++) {
4999 const struct rte_flow_action_queue *queue;
5000 const struct rte_flow_action_rss *rss;
5001 const struct rte_flow_action *action = actions;
5002 const struct rte_flow_action_count *count = action->conf;
5003 const uint8_t *rss_key;
5004 const struct rte_flow_action_jump *jump_data;
5005 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
5006 struct mlx5_flow_tbl_resource *tbl;
5007 uint32_t port_id = 0;
5008 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
5010 switch (actions->type) {
5011 case RTE_FLOW_ACTION_TYPE_VOID:
5013 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5014 if (flow_dv_translate_action_port_id(dev, action,
5017 port_id_resource.port_id = port_id;
5018 if (flow_dv_port_id_action_resource_register
5019 (dev, &port_id_resource, dev_flow, error))
5021 dev_flow->dv.actions[actions_n++] =
5022 dev_flow->dv.port_id_action->action;
5023 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5025 case RTE_FLOW_ACTION_TYPE_FLAG:
5027 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
5028 if (!flow->tag_resource)
5029 if (flow_dv_tag_resource_register
5030 (dev, &tag_resource, dev_flow, error))
5032 dev_flow->dv.actions[actions_n++] =
5033 flow->tag_resource->action;
5034 action_flags |= MLX5_FLOW_ACTION_FLAG;
5036 case RTE_FLOW_ACTION_TYPE_MARK:
5037 tag_resource.tag = mlx5_flow_mark_set
5038 (((const struct rte_flow_action_mark *)
5039 (actions->conf))->id);
5040 if (!flow->tag_resource)
5041 if (flow_dv_tag_resource_register
5042 (dev, &tag_resource, dev_flow, error))
5044 dev_flow->dv.actions[actions_n++] =
5045 flow->tag_resource->action;
5046 action_flags |= MLX5_FLOW_ACTION_MARK;
5048 case RTE_FLOW_ACTION_TYPE_DROP:
5049 action_flags |= MLX5_FLOW_ACTION_DROP;
5051 case RTE_FLOW_ACTION_TYPE_QUEUE:
5052 queue = actions->conf;
5053 flow->rss.queue_num = 1;
5054 (*flow->queue)[0] = queue->index;
5055 action_flags |= MLX5_FLOW_ACTION_QUEUE;
5057 case RTE_FLOW_ACTION_TYPE_RSS:
5058 rss = actions->conf;
5060 memcpy((*flow->queue), rss->queue,
5061 rss->queue_num * sizeof(uint16_t));
5062 flow->rss.queue_num = rss->queue_num;
5063 /* NULL RSS key indicates default RSS key. */
5064 rss_key = !rss->key ? rss_hash_default_key : rss->key;
5065 memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
5066 /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
5067 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
5068 flow->rss.level = rss->level;
5069 action_flags |= MLX5_FLOW_ACTION_RSS;
5071 case RTE_FLOW_ACTION_TYPE_COUNT:
5072 if (!priv->config.devx) {
5073 rte_errno = ENOTSUP;
5076 flow->counter = flow_dv_counter_alloc(dev,
5080 if (flow->counter == NULL)
5082 dev_flow->dv.actions[actions_n++] =
5083 flow->counter->action;
5084 action_flags |= MLX5_FLOW_ACTION_COUNT;
5087 if (rte_errno == ENOTSUP)
5088 return rte_flow_error_set
5090 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5092 "count action not supported");
5094 return rte_flow_error_set
5096 RTE_FLOW_ERROR_TYPE_ACTION,
5098 "cannot create counter"
5101 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5102 dev_flow->dv.actions[actions_n++] =
5103 priv->sh->pop_vlan_action;
5104 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5106 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5107 if (!vlan_inherited) {
5108 flow_dev_get_vlan_info_from_items(items, &vlan);
5109 vlan_inherited = true;
5111 vlan.eth_proto = rte_be_to_cpu_16
5112 ((((const struct rte_flow_action_of_push_vlan *)
5113 actions->conf)->ethertype));
5114 if (flow_dv_create_action_push_vlan
5115 (dev, attr, &vlan, dev_flow, error))
5117 dev_flow->dv.actions[actions_n++] =
5118 dev_flow->dv.push_vlan_res->action;
5119 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5121 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5122 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5123 if (flow_dv_create_action_l2_encap(dev, actions,
5128 dev_flow->dv.actions[actions_n++] =
5129 dev_flow->dv.encap_decap->verbs_action;
5130 action_flags |= actions->type ==
5131 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
5132 MLX5_FLOW_ACTION_VXLAN_ENCAP :
5133 MLX5_FLOW_ACTION_NVGRE_ENCAP;
5135 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5136 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5137 if (flow_dv_create_action_l2_decap(dev, dev_flow,
5141 dev_flow->dv.actions[actions_n++] =
5142 dev_flow->dv.encap_decap->verbs_action;
5143 action_flags |= actions->type ==
5144 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
5145 MLX5_FLOW_ACTION_VXLAN_DECAP :
5146 MLX5_FLOW_ACTION_NVGRE_DECAP;
5148 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5149 /* Handle encap with preceding decap. */
5150 if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
5151 if (flow_dv_create_action_raw_encap
5152 (dev, actions, dev_flow, attr, error))
5154 dev_flow->dv.actions[actions_n++] =
5155 dev_flow->dv.encap_decap->verbs_action;
5157 /* Handle encap without preceding decap. */
5158 if (flow_dv_create_action_l2_encap
5159 (dev, actions, dev_flow, attr->transfer,
5162 dev_flow->dv.actions[actions_n++] =
5163 dev_flow->dv.encap_decap->verbs_action;
5165 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
5167 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5168 /* Check if this decap is followed by encap. */
5169 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
5170 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
5173 /* Handle decap only if it isn't followed by encap. */
5174 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5175 if (flow_dv_create_action_l2_decap
5176 (dev, dev_flow, attr->transfer, error))
5178 dev_flow->dv.actions[actions_n++] =
5179 dev_flow->dv.encap_decap->verbs_action;
5181 /* If decap is followed by encap, handle it at encap. */
5182 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
5184 case RTE_FLOW_ACTION_TYPE_JUMP:
5185 jump_data = action->conf;
5186 tbl = flow_dv_tbl_resource_get(dev, jump_data->group,
5188 attr->transfer, error);
5190 return rte_flow_error_set
5192 RTE_FLOW_ERROR_TYPE_ACTION,
5194 "cannot create jump action.");
5195 jump_tbl_resource.tbl = tbl;
5196 if (flow_dv_jump_tbl_resource_register
5197 (dev, &jump_tbl_resource, dev_flow, error)) {
5198 flow_dv_tbl_resource_release(tbl);
5199 return rte_flow_error_set
5201 RTE_FLOW_ERROR_TYPE_ACTION,
5203 "cannot create jump action.");
5205 dev_flow->dv.actions[actions_n++] =
5206 dev_flow->dv.jump->action;
5207 action_flags |= MLX5_FLOW_ACTION_JUMP;
5209 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5210 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5211 if (flow_dv_convert_action_modify_mac(&res, actions,
5214 action_flags |= actions->type ==
5215 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5216 MLX5_FLOW_ACTION_SET_MAC_SRC :
5217 MLX5_FLOW_ACTION_SET_MAC_DST;
5219 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5220 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5221 if (flow_dv_convert_action_modify_ipv4(&res, actions,
5224 action_flags |= actions->type ==
5225 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5226 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5227 MLX5_FLOW_ACTION_SET_IPV4_DST;
5229 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5230 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5231 if (flow_dv_convert_action_modify_ipv6(&res, actions,
5234 action_flags |= actions->type ==
5235 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5236 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5237 MLX5_FLOW_ACTION_SET_IPV6_DST;
5239 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5240 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5241 if (flow_dv_convert_action_modify_tp(&res, actions,
5245 action_flags |= actions->type ==
5246 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5247 MLX5_FLOW_ACTION_SET_TP_SRC :
5248 MLX5_FLOW_ACTION_SET_TP_DST;
5250 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5251 if (flow_dv_convert_action_modify_dec_ttl(&res, items,
5255 action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
5257 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5258 if (flow_dv_convert_action_modify_ttl(&res, actions,
5262 action_flags |= MLX5_FLOW_ACTION_SET_TTL;
5264 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5265 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5266 if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
5269 action_flags |= actions->type ==
5270 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5271 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5272 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5275 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5276 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5277 if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
5280 action_flags |= actions->type ==
5281 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5282 MLX5_FLOW_ACTION_INC_TCP_ACK :
5283 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5285 case RTE_FLOW_ACTION_TYPE_END:
5287 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
5288 /* create modify action if needed. */
5289 if (flow_dv_modify_hdr_resource_register
5294 dev_flow->dv.actions[modify_action_position] =
5295 dev_flow->dv.modify_hdr->verbs_action;
5301 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
5302 modify_action_position == UINT32_MAX)
5303 modify_action_position = actions_n++;
5305 dev_flow->dv.actions_n = actions_n;
5306 flow->actions = action_flags;
5307 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5308 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5310 switch (items->type) {
5311 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5312 flow_dv_translate_item_port_id(dev, match_mask,
5313 match_value, items);
5314 last_item = MLX5_FLOW_ITEM_PORT_ID;
5316 case RTE_FLOW_ITEM_TYPE_ETH:
5317 flow_dv_translate_item_eth(match_mask, match_value,
5319 matcher.priority = MLX5_PRIORITY_MAP_L2;
5320 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5321 MLX5_FLOW_LAYER_OUTER_L2;
5323 case RTE_FLOW_ITEM_TYPE_VLAN:
5324 flow_dv_translate_item_vlan(dev_flow,
5325 match_mask, match_value,
5327 matcher.priority = MLX5_PRIORITY_MAP_L2;
5328 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
5329 MLX5_FLOW_LAYER_INNER_VLAN) :
5330 (MLX5_FLOW_LAYER_OUTER_L2 |
5331 MLX5_FLOW_LAYER_OUTER_VLAN);
5333 case RTE_FLOW_ITEM_TYPE_IPV4:
5334 mlx5_flow_tunnel_ip_check(items, next_protocol,
5335 &item_flags, &tunnel);
5336 flow_dv_translate_item_ipv4(match_mask, match_value,
5337 items, tunnel, attr->group);
5338 matcher.priority = MLX5_PRIORITY_MAP_L3;
5339 dev_flow->dv.hash_fields |=
5340 mlx5_flow_hashfields_adjust
5342 MLX5_IPV4_LAYER_TYPES,
5343 MLX5_IPV4_IBV_RX_HASH);
5344 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5345 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5346 if (items->mask != NULL &&
5347 ((const struct rte_flow_item_ipv4 *)
5348 items->mask)->hdr.next_proto_id) {
5350 ((const struct rte_flow_item_ipv4 *)
5351 (items->spec))->hdr.next_proto_id;
5353 ((const struct rte_flow_item_ipv4 *)
5354 (items->mask))->hdr.next_proto_id;
5356 /* Reset for inner layer. */
5357 next_protocol = 0xff;
5360 case RTE_FLOW_ITEM_TYPE_IPV6:
5361 mlx5_flow_tunnel_ip_check(items, next_protocol,
5362 &item_flags, &tunnel);
5363 flow_dv_translate_item_ipv6(match_mask, match_value,
5364 items, tunnel, attr->group);
5365 matcher.priority = MLX5_PRIORITY_MAP_L3;
5366 dev_flow->dv.hash_fields |=
5367 mlx5_flow_hashfields_adjust
5369 MLX5_IPV6_LAYER_TYPES,
5370 MLX5_IPV6_IBV_RX_HASH);
5371 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5372 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5373 if (items->mask != NULL &&
5374 ((const struct rte_flow_item_ipv6 *)
5375 items->mask)->hdr.proto) {
5377 ((const struct rte_flow_item_ipv6 *)
5378 items->spec)->hdr.proto;
5380 ((const struct rte_flow_item_ipv6 *)
5381 items->mask)->hdr.proto;
5383 /* Reset for inner layer. */
5384 next_protocol = 0xff;
5387 case RTE_FLOW_ITEM_TYPE_TCP:
5388 flow_dv_translate_item_tcp(match_mask, match_value,
5390 matcher.priority = MLX5_PRIORITY_MAP_L4;
5391 dev_flow->dv.hash_fields |=
5392 mlx5_flow_hashfields_adjust
5393 (dev_flow, tunnel, ETH_RSS_TCP,
5394 IBV_RX_HASH_SRC_PORT_TCP |
5395 IBV_RX_HASH_DST_PORT_TCP);
5396 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5397 MLX5_FLOW_LAYER_OUTER_L4_TCP;
5399 case RTE_FLOW_ITEM_TYPE_UDP:
5400 flow_dv_translate_item_udp(match_mask, match_value,
5402 matcher.priority = MLX5_PRIORITY_MAP_L4;
5403 dev_flow->dv.hash_fields |=
5404 mlx5_flow_hashfields_adjust
5405 (dev_flow, tunnel, ETH_RSS_UDP,
5406 IBV_RX_HASH_SRC_PORT_UDP |
5407 IBV_RX_HASH_DST_PORT_UDP);
5408 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5409 MLX5_FLOW_LAYER_OUTER_L4_UDP;
5411 case RTE_FLOW_ITEM_TYPE_GRE:
5412 flow_dv_translate_item_gre(match_mask, match_value,
5414 last_item = MLX5_FLOW_LAYER_GRE;
5416 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5417 flow_dv_translate_item_gre_key(match_mask,
5418 match_value, items);
5419 last_item = MLX5_FLOW_LAYER_GRE_KEY;
5421 case RTE_FLOW_ITEM_TYPE_NVGRE:
5422 flow_dv_translate_item_nvgre(match_mask, match_value,
5424 last_item = MLX5_FLOW_LAYER_GRE;
5426 case RTE_FLOW_ITEM_TYPE_VXLAN:
5427 flow_dv_translate_item_vxlan(match_mask, match_value,
5429 last_item = MLX5_FLOW_LAYER_VXLAN;
5431 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5432 flow_dv_translate_item_vxlan(match_mask, match_value,
5434 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5436 case RTE_FLOW_ITEM_TYPE_MPLS:
5437 flow_dv_translate_item_mpls(match_mask, match_value,
5438 items, last_item, tunnel);
5439 last_item = MLX5_FLOW_LAYER_MPLS;
5441 case RTE_FLOW_ITEM_TYPE_META:
5442 flow_dv_translate_item_meta(match_mask, match_value,
5444 last_item = MLX5_FLOW_ITEM_METADATA;
5446 case RTE_FLOW_ITEM_TYPE_ICMP:
5447 flow_dv_translate_item_icmp(match_mask, match_value,
5449 last_item = MLX5_FLOW_LAYER_ICMP;
5451 case RTE_FLOW_ITEM_TYPE_ICMP6:
5452 flow_dv_translate_item_icmp6(match_mask, match_value,
5454 last_item = MLX5_FLOW_LAYER_ICMP6;
5459 item_flags |= last_item;
5462 * In case of ingress traffic when E-Switch mode is enabled,
5463 * we have two cases where we need to set the source port manually.
5464 * The first one, is in case of Nic steering rule, and the second is
5465 * E-Switch rule where no port_id item was found. In both cases
5466 * the source port is set according the current port in use.
5468 if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
5469 (priv->representor || priv->master)) {
5470 if (flow_dv_translate_item_port_id(dev, match_mask,
5474 assert(!flow_dv_check_valid_spec(matcher.mask.buf,
5475 dev_flow->dv.value.buf));
5476 dev_flow->layers = item_flags;
5477 /* Register matcher. */
5478 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
5480 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
5482 matcher.egress = attr->egress;
5483 matcher.group = attr->group;
5484 matcher.transfer = attr->transfer;
5485 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
5491 * Apply the flow to the NIC.
5494 * Pointer to the Ethernet device structure.
5495 * @param[in, out] flow
5496 * Pointer to flow structure.
5498 * Pointer to error structure.
5501 * 0 on success, a negative errno value otherwise and rte_errno is set.
5504 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
5505 struct rte_flow_error *error)
5507 struct mlx5_flow_dv *dv;
5508 struct mlx5_flow *dev_flow;
5509 struct mlx5_priv *priv = dev->data->dev_private;
5513 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5516 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
5517 if (flow->transfer) {
5518 dv->actions[n++] = priv->sh->esw_drop_action;
5520 dv->hrxq = mlx5_hrxq_drop_new(dev);
5524 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5526 "cannot get drop hash queue");
5529 dv->actions[n++] = dv->hrxq->action;
5531 } else if (flow->actions &
5532 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
5533 struct mlx5_hrxq *hrxq;
5535 hrxq = mlx5_hrxq_get(dev, flow->key,
5536 MLX5_RSS_HASH_KEY_LEN,
5539 flow->rss.queue_num);
5541 hrxq = mlx5_hrxq_new
5542 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
5543 dv->hash_fields, (*flow->queue),
5544 flow->rss.queue_num,
5545 !!(dev_flow->layers &
5546 MLX5_FLOW_LAYER_TUNNEL));
5551 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5552 "cannot get hash queue");
5556 dv->actions[n++] = dv->hrxq->action;
5559 mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
5560 (void *)&dv->value, n,
5563 rte_flow_error_set(error, errno,
5564 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5566 "hardware refuses to create flow");
5569 if (priv->vmwa_context &&
5570 dev_flow->dv.vf_vlan.tag &&
5571 !dev_flow->dv.vf_vlan.created) {
5573 * The rule contains the VLAN pattern.
5574 * For VF we are going to create VLAN
5575 * interface to make hypervisor set correct
5576 * e-Switch vport context.
5578 mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan);
5583 err = rte_errno; /* Save rte_errno before cleanup. */
5584 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5585 struct mlx5_flow_dv *dv = &dev_flow->dv;
5587 if (flow->actions & MLX5_FLOW_ACTION_DROP)
5588 mlx5_hrxq_drop_release(dev);
5590 mlx5_hrxq_release(dev, dv->hrxq);
5593 if (dev_flow->dv.vf_vlan.tag &&
5594 dev_flow->dv.vf_vlan.created)
5595 mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5597 rte_errno = err; /* Restore rte_errno. */
5602 * Release the flow matcher.
5605 * Pointer to Ethernet device.
5607 * Pointer to mlx5_flow.
5610 * 1 while a reference on it exists, 0 when freed.
5613 flow_dv_matcher_release(struct rte_eth_dev *dev,
5614 struct mlx5_flow *flow)
5616 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
5617 struct mlx5_priv *priv = dev->data->dev_private;
5618 struct mlx5_ibv_shared *sh = priv->sh;
5619 struct mlx5_flow_tbl_resource *tbl;
5621 assert(matcher->matcher_object);
5622 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
5623 dev->data->port_id, (void *)matcher,
5624 rte_atomic32_read(&matcher->refcnt));
5625 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
5626 claim_zero(mlx5_glue->dv_destroy_flow_matcher
5627 (matcher->matcher_object));
5628 LIST_REMOVE(matcher, next);
5629 if (matcher->egress)
5630 tbl = &sh->tx_tbl[matcher->group];
5632 tbl = &sh->rx_tbl[matcher->group];
5633 flow_dv_tbl_resource_release(tbl);
5635 DRV_LOG(DEBUG, "port %u matcher %p: removed",
5636 dev->data->port_id, (void *)matcher);
5643 * Release an encap/decap resource.
5646 * Pointer to mlx5_flow.
5649 * 1 while a reference on it exists, 0 when freed.
5652 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
5654 struct mlx5_flow_dv_encap_decap_resource *cache_resource =
5655 flow->dv.encap_decap;
5657 assert(cache_resource->verbs_action);
5658 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
5659 (void *)cache_resource,
5660 rte_atomic32_read(&cache_resource->refcnt));
5661 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5662 claim_zero(mlx5_glue->destroy_flow_action
5663 (cache_resource->verbs_action));
5664 LIST_REMOVE(cache_resource, next);
5665 rte_free(cache_resource);
5666 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
5667 (void *)cache_resource);
5674 * Release an jump to table action resource.
5677 * Pointer to mlx5_flow.
5680 * 1 while a reference on it exists, 0 when freed.
5683 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
5685 struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
5688 assert(cache_resource->action);
5689 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
5690 (void *)cache_resource,
5691 rte_atomic32_read(&cache_resource->refcnt));
5692 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5693 claim_zero(mlx5_glue->destroy_flow_action
5694 (cache_resource->action));
5695 LIST_REMOVE(cache_resource, next);
5696 flow_dv_tbl_resource_release(cache_resource->tbl);
5697 rte_free(cache_resource);
5698 DRV_LOG(DEBUG, "jump table resource %p: removed",
5699 (void *)cache_resource);
5706 * Release a modify-header resource.
5709 * Pointer to mlx5_flow.
5712 * 1 while a reference on it exists, 0 when freed.
5715 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
5717 struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
5718 flow->dv.modify_hdr;
5720 assert(cache_resource->verbs_action);
5721 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
5722 (void *)cache_resource,
5723 rte_atomic32_read(&cache_resource->refcnt));
5724 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5725 claim_zero(mlx5_glue->destroy_flow_action
5726 (cache_resource->verbs_action));
5727 LIST_REMOVE(cache_resource, next);
5728 rte_free(cache_resource);
5729 DRV_LOG(DEBUG, "modify-header resource %p: removed",
5730 (void *)cache_resource);
5737 * Release port ID action resource.
5740 * Pointer to mlx5_flow.
5743 * 1 while a reference on it exists, 0 when freed.
5746 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
5748 struct mlx5_flow_dv_port_id_action_resource *cache_resource =
5749 flow->dv.port_id_action;
5751 assert(cache_resource->action);
5752 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
5753 (void *)cache_resource,
5754 rte_atomic32_read(&cache_resource->refcnt));
5755 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5756 claim_zero(mlx5_glue->destroy_flow_action
5757 (cache_resource->action));
5758 LIST_REMOVE(cache_resource, next);
5759 rte_free(cache_resource);
5760 DRV_LOG(DEBUG, "port id action resource %p: removed",
5761 (void *)cache_resource);
5768 * Release push vlan action resource.
5771 * Pointer to mlx5_flow.
5774 * 1 while a reference on it exists, 0 when freed.
5777 flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow)
5779 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource =
5780 flow->dv.push_vlan_res;
5782 assert(cache_resource->action);
5783 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
5784 (void *)cache_resource,
5785 rte_atomic32_read(&cache_resource->refcnt));
5786 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
5787 claim_zero(mlx5_glue->destroy_flow_action
5788 (cache_resource->action));
5789 LIST_REMOVE(cache_resource, next);
5790 rte_free(cache_resource);
5791 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
5792 (void *)cache_resource);
5799 * Remove the flow from the NIC but keeps it in memory.
5802 * Pointer to Ethernet device.
5803 * @param[in, out] flow
5804 * Pointer to flow structure.
5807 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
5809 struct mlx5_flow_dv *dv;
5810 struct mlx5_flow *dev_flow;
5814 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
5817 claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
5821 if (flow->actions & MLX5_FLOW_ACTION_DROP)
5822 mlx5_hrxq_drop_release(dev);
5824 mlx5_hrxq_release(dev, dv->hrxq);
5827 if (dev_flow->dv.vf_vlan.tag &&
5828 dev_flow->dv.vf_vlan.created)
5829 mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan);
5834 * Remove the flow from the NIC and the memory.
5837 * Pointer to the Ethernet device structure.
5838 * @param[in, out] flow
5839 * Pointer to flow structure.
5842 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
5844 struct mlx5_flow *dev_flow;
5848 flow_dv_remove(dev, flow);
5849 if (flow->counter) {
5850 flow_dv_counter_release(dev, flow->counter);
5851 flow->counter = NULL;
5853 if (flow->tag_resource) {
5854 flow_dv_tag_release(dev, flow->tag_resource);
5855 flow->tag_resource = NULL;
5857 while (!LIST_EMPTY(&flow->dev_flows)) {
5858 dev_flow = LIST_FIRST(&flow->dev_flows);
5859 LIST_REMOVE(dev_flow, next);
5860 if (dev_flow->dv.matcher)
5861 flow_dv_matcher_release(dev, dev_flow);
5862 if (dev_flow->dv.encap_decap)
5863 flow_dv_encap_decap_resource_release(dev_flow);
5864 if (dev_flow->dv.modify_hdr)
5865 flow_dv_modify_hdr_resource_release(dev_flow);
5866 if (dev_flow->dv.jump)
5867 flow_dv_jump_tbl_resource_release(dev_flow);
5868 if (dev_flow->dv.port_id_action)
5869 flow_dv_port_id_action_resource_release(dev_flow);
5870 if (dev_flow->dv.push_vlan_res)
5871 flow_dv_push_vlan_action_resource_release(dev_flow);
5877 * Query a dv flow rule for its statistics via devx.
5880 * Pointer to Ethernet device.
5882 * Pointer to the sub flow.
5884 * data retrieved by the query.
5886 * Perform verbose error reporting if not NULL.
5889 * 0 on success, a negative errno value otherwise and rte_errno is set.
5892 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
5893 void *data, struct rte_flow_error *error)
5895 struct mlx5_priv *priv = dev->data->dev_private;
5896 struct rte_flow_query_count *qc = data;
5898 if (!priv->config.devx)
5899 return rte_flow_error_set(error, ENOTSUP,
5900 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5902 "counters are not supported");
5903 if (flow->counter) {
5904 uint64_t pkts, bytes;
5905 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
5909 return rte_flow_error_set(error, -err,
5910 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5911 NULL, "cannot read counters");
5914 qc->hits = pkts - flow->counter->hits;
5915 qc->bytes = bytes - flow->counter->bytes;
5917 flow->counter->hits = pkts;
5918 flow->counter->bytes = bytes;
5922 return rte_flow_error_set(error, EINVAL,
5923 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5925 "counters are not available");
5931 * @see rte_flow_query()
5935 flow_dv_query(struct rte_eth_dev *dev,
5936 struct rte_flow *flow __rte_unused,
5937 const struct rte_flow_action *actions __rte_unused,
5938 void *data __rte_unused,
5939 struct rte_flow_error *error __rte_unused)
5943 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5944 switch (actions->type) {
5945 case RTE_FLOW_ACTION_TYPE_VOID:
5947 case RTE_FLOW_ACTION_TYPE_COUNT:
5948 ret = flow_dv_query_count(dev, flow, data, error);
5951 return rte_flow_error_set(error, ENOTSUP,
5952 RTE_FLOW_ERROR_TYPE_ACTION,
5954 "action not supported");
5961 * Mutex-protected thunk to flow_dv_translate().
5964 flow_d_translate(struct rte_eth_dev *dev,
5965 struct mlx5_flow *dev_flow,
5966 const struct rte_flow_attr *attr,
5967 const struct rte_flow_item items[],
5968 const struct rte_flow_action actions[],
5969 struct rte_flow_error *error)
5973 flow_d_shared_lock(dev);
5974 ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
5975 flow_d_shared_unlock(dev);
5980 * Mutex-protected thunk to flow_dv_apply().
5983 flow_d_apply(struct rte_eth_dev *dev,
5984 struct rte_flow *flow,
5985 struct rte_flow_error *error)
5989 flow_d_shared_lock(dev);
5990 ret = flow_dv_apply(dev, flow, error);
5991 flow_d_shared_unlock(dev);
5996 * Mutex-protected thunk to flow_dv_remove().
5999 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
6001 flow_d_shared_lock(dev);
6002 flow_dv_remove(dev, flow);
6003 flow_d_shared_unlock(dev);
6007 * Mutex-protected thunk to flow_dv_destroy().
6010 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
6012 flow_d_shared_lock(dev);
6013 flow_dv_destroy(dev, flow);
6014 flow_d_shared_unlock(dev);
6017 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
6018 .validate = flow_dv_validate,
6019 .prepare = flow_dv_prepare,
6020 .translate = flow_d_translate,
6021 .apply = flow_d_apply,
6022 .remove = flow_d_remove,
6023 .destroy = flow_d_destroy,
6024 .query = flow_dv_query,
6027 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */