1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
27 #include <rte_cycles.h>
30 #include <rte_vxlan.h>
33 #include <mlx5_devx_cmds.h>
36 #include "mlx5_defs.h"
38 #include "mlx5_common_os.h"
39 #include "mlx5_flow.h"
40 #include "mlx5_flow_os.h"
41 #include "mlx5_rxtx.h"
43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
49 #ifndef HAVE_MLX5DV_DR_ESWITCH
50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
55 #ifndef HAVE_MLX5DV_DR
56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
59 /* VLAN header definitions */
60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
79 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
80 struct mlx5_flow_tbl_resource *tbl);
83 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
86 * Initialize flow attributes structure according to flow items' types.
88 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
89 * mode. For tunnel mode, the items to be modified are the outermost ones.
92 * Pointer to item specification.
94 * Pointer to flow attributes structure.
96 * Pointer to the sub flow.
97 * @param[in] tunnel_decap
98 * Whether action is after tunnel decapsulation.
101 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
102 struct mlx5_flow *dev_flow, bool tunnel_decap)
104 uint64_t layers = dev_flow->handle->layers;
107 * If layers is already initialized, it means this dev_flow is the
108 * suffix flow, the layers flags is set by the prefix flow. Need to
109 * use the layer flags from prefix flow as the suffix flow may not
110 * have the user defined items as the flow is split.
113 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
115 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
117 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
119 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
124 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
125 uint8_t next_protocol = 0xff;
126 switch (item->type) {
127 case RTE_FLOW_ITEM_TYPE_GRE:
128 case RTE_FLOW_ITEM_TYPE_NVGRE:
129 case RTE_FLOW_ITEM_TYPE_VXLAN:
130 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
131 case RTE_FLOW_ITEM_TYPE_GENEVE:
132 case RTE_FLOW_ITEM_TYPE_MPLS:
136 case RTE_FLOW_ITEM_TYPE_IPV4:
139 if (item->mask != NULL &&
140 ((const struct rte_flow_item_ipv4 *)
141 item->mask)->hdr.next_proto_id)
143 ((const struct rte_flow_item_ipv4 *)
144 (item->spec))->hdr.next_proto_id &
145 ((const struct rte_flow_item_ipv4 *)
146 (item->mask))->hdr.next_proto_id;
147 if ((next_protocol == IPPROTO_IPIP ||
148 next_protocol == IPPROTO_IPV6) && tunnel_decap)
151 case RTE_FLOW_ITEM_TYPE_IPV6:
154 if (item->mask != NULL &&
155 ((const struct rte_flow_item_ipv6 *)
156 item->mask)->hdr.proto)
158 ((const struct rte_flow_item_ipv6 *)
159 (item->spec))->hdr.proto &
160 ((const struct rte_flow_item_ipv6 *)
161 (item->mask))->hdr.proto;
162 if ((next_protocol == IPPROTO_IPIP ||
163 next_protocol == IPPROTO_IPV6) && tunnel_decap)
166 case RTE_FLOW_ITEM_TYPE_UDP:
170 case RTE_FLOW_ITEM_TYPE_TCP:
182 * Convert rte_mtr_color to mlx5 color.
191 rte_col_2_mlx5_col(enum rte_color rcol)
194 case RTE_COLOR_GREEN:
195 return MLX5_FLOW_COLOR_GREEN;
196 case RTE_COLOR_YELLOW:
197 return MLX5_FLOW_COLOR_YELLOW;
199 return MLX5_FLOW_COLOR_RED;
203 return MLX5_FLOW_COLOR_UNDEFINED;
206 struct field_modify_info {
207 uint32_t size; /* Size of field in protocol header, in bytes. */
208 uint32_t offset; /* Offset of field in protocol header, in bytes. */
209 enum mlx5_modification_field id;
212 struct field_modify_info modify_eth[] = {
213 {4, 0, MLX5_MODI_OUT_DMAC_47_16},
214 {2, 4, MLX5_MODI_OUT_DMAC_15_0},
215 {4, 6, MLX5_MODI_OUT_SMAC_47_16},
216 {2, 10, MLX5_MODI_OUT_SMAC_15_0},
220 struct field_modify_info modify_vlan_out_first_vid[] = {
221 /* Size in bits !!! */
222 {12, 0, MLX5_MODI_OUT_FIRST_VID},
226 struct field_modify_info modify_ipv4[] = {
227 {1, 1, MLX5_MODI_OUT_IP_DSCP},
228 {1, 8, MLX5_MODI_OUT_IPV4_TTL},
229 {4, 12, MLX5_MODI_OUT_SIPV4},
230 {4, 16, MLX5_MODI_OUT_DIPV4},
234 struct field_modify_info modify_ipv6[] = {
235 {1, 0, MLX5_MODI_OUT_IP_DSCP},
236 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
237 {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
238 {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
239 {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
240 {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
241 {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
242 {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
243 {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
244 {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
248 struct field_modify_info modify_udp[] = {
249 {2, 0, MLX5_MODI_OUT_UDP_SPORT},
250 {2, 2, MLX5_MODI_OUT_UDP_DPORT},
254 struct field_modify_info modify_tcp[] = {
255 {2, 0, MLX5_MODI_OUT_TCP_SPORT},
256 {2, 2, MLX5_MODI_OUT_TCP_DPORT},
257 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
258 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
263 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
264 uint8_t next_protocol, uint64_t *item_flags,
267 MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
268 item->type == RTE_FLOW_ITEM_TYPE_IPV6);
269 if (next_protocol == IPPROTO_IPIP) {
270 *item_flags |= MLX5_FLOW_LAYER_IPIP;
273 if (next_protocol == IPPROTO_IPV6) {
274 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
280 * Acquire the synchronizing object to protect multithreaded access
281 * to shared dv context. Lock occurs only if context is actually
282 * shared, i.e. we have multiport IB device and representors are
286 * Pointer to the rte_eth_dev structure.
289 flow_dv_shared_lock(struct rte_eth_dev *dev)
291 struct mlx5_priv *priv = dev->data->dev_private;
292 struct mlx5_dev_ctx_shared *sh = priv->sh;
294 if (sh->dv_refcnt > 1) {
297 ret = pthread_mutex_lock(&sh->dv_mutex);
304 flow_dv_shared_unlock(struct rte_eth_dev *dev)
306 struct mlx5_priv *priv = dev->data->dev_private;
307 struct mlx5_dev_ctx_shared *sh = priv->sh;
309 if (sh->dv_refcnt > 1) {
312 ret = pthread_mutex_unlock(&sh->dv_mutex);
318 /* Update VLAN's VID/PCP based on input rte_flow_action.
321 * Pointer to struct rte_flow_action.
323 * Pointer to struct rte_vlan_hdr.
326 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
327 struct rte_vlan_hdr *vlan)
330 if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
332 ((const struct rte_flow_action_of_set_vlan_pcp *)
333 action->conf)->vlan_pcp;
334 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
335 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
336 vlan->vlan_tci |= vlan_tci;
337 } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
338 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
339 vlan->vlan_tci |= rte_be_to_cpu_16
340 (((const struct rte_flow_action_of_set_vlan_vid *)
341 action->conf)->vlan_vid);
346 * Fetch 1, 2, 3 or 4 byte field from the byte array
347 * and return as unsigned integer in host-endian format.
350 * Pointer to data array.
352 * Size of field to extract.
355 * converted field in host endian format.
357 static inline uint32_t
358 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
367 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
370 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
371 ret = (ret << 8) | *(data + sizeof(uint16_t));
374 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
385 * Convert modify-header action to DV specification.
387 * Data length of each action is determined by provided field description
388 * and the item mask. Data bit offset and width of each action is determined
389 * by provided item mask.
392 * Pointer to item specification.
394 * Pointer to field modification information.
395 * For MLX5_MODIFICATION_TYPE_SET specifies destination field.
396 * For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
397 * For MLX5_MODIFICATION_TYPE_COPY specifies source field.
399 * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
400 * Negative offset value sets the same offset as source offset.
401 * size field is ignored, value is taken from source field.
402 * @param[in,out] resource
403 * Pointer to the modify-header resource.
405 * Type of modification.
407 * Pointer to the error structure.
410 * 0 on success, a negative errno value otherwise and rte_errno is set.
413 flow_dv_convert_modify_action(struct rte_flow_item *item,
414 struct field_modify_info *field,
415 struct field_modify_info *dcopy,
416 struct mlx5_flow_dv_modify_hdr_resource *resource,
417 uint32_t type, struct rte_flow_error *error)
419 uint32_t i = resource->actions_num;
420 struct mlx5_modification_cmd *actions = resource->actions;
423 * The item and mask are provided in big-endian format.
424 * The fields should be presented as in big-endian format either.
425 * Mask must be always present, it defines the actual field width.
427 MLX5_ASSERT(item->mask);
428 MLX5_ASSERT(field->size);
435 if (i >= MLX5_MAX_MODIFY_NUM)
436 return rte_flow_error_set(error, EINVAL,
437 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
438 "too many items to modify");
439 /* Fetch variable byte size mask from the array. */
440 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
441 field->offset, field->size);
446 /* Deduce actual data width in bits from mask value. */
447 off_b = rte_bsf32(mask);
448 size_b = sizeof(uint32_t) * CHAR_BIT -
449 off_b - __builtin_clz(mask);
451 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
452 actions[i] = (struct mlx5_modification_cmd) {
458 /* Convert entire record to expected big-endian format. */
459 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
460 if (type == MLX5_MODIFICATION_TYPE_COPY) {
462 actions[i].dst_field = dcopy->id;
463 actions[i].dst_offset =
464 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
465 /* Convert entire record to big-endian format. */
466 actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
468 MLX5_ASSERT(item->spec);
469 data = flow_dv_fetch_field((const uint8_t *)item->spec +
470 field->offset, field->size);
471 /* Shift out the trailing masked bits from data. */
472 data = (data & mask) >> off_b;
473 actions[i].data1 = rte_cpu_to_be_32(data);
477 } while (field->size);
478 if (resource->actions_num == i)
479 return rte_flow_error_set(error, EINVAL,
480 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
481 "invalid modification flow item");
482 resource->actions_num = i;
487 * Convert modify-header set IPv4 address action to DV specification.
489 * @param[in,out] resource
490 * Pointer to the modify-header resource.
492 * Pointer to action specification.
494 * Pointer to the error structure.
497 * 0 on success, a negative errno value otherwise and rte_errno is set.
500 flow_dv_convert_action_modify_ipv4
501 (struct mlx5_flow_dv_modify_hdr_resource *resource,
502 const struct rte_flow_action *action,
503 struct rte_flow_error *error)
505 const struct rte_flow_action_set_ipv4 *conf =
506 (const struct rte_flow_action_set_ipv4 *)(action->conf);
507 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
508 struct rte_flow_item_ipv4 ipv4;
509 struct rte_flow_item_ipv4 ipv4_mask;
511 memset(&ipv4, 0, sizeof(ipv4));
512 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
513 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
514 ipv4.hdr.src_addr = conf->ipv4_addr;
515 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
517 ipv4.hdr.dst_addr = conf->ipv4_addr;
518 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
521 item.mask = &ipv4_mask;
522 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
523 MLX5_MODIFICATION_TYPE_SET, error);
527 * Convert modify-header set IPv6 address action to DV specification.
529 * @param[in,out] resource
530 * Pointer to the modify-header resource.
532 * Pointer to action specification.
534 * Pointer to the error structure.
537 * 0 on success, a negative errno value otherwise and rte_errno is set.
540 flow_dv_convert_action_modify_ipv6
541 (struct mlx5_flow_dv_modify_hdr_resource *resource,
542 const struct rte_flow_action *action,
543 struct rte_flow_error *error)
545 const struct rte_flow_action_set_ipv6 *conf =
546 (const struct rte_flow_action_set_ipv6 *)(action->conf);
547 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
548 struct rte_flow_item_ipv6 ipv6;
549 struct rte_flow_item_ipv6 ipv6_mask;
551 memset(&ipv6, 0, sizeof(ipv6));
552 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
553 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
554 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
555 sizeof(ipv6.hdr.src_addr));
556 memcpy(&ipv6_mask.hdr.src_addr,
557 &rte_flow_item_ipv6_mask.hdr.src_addr,
558 sizeof(ipv6.hdr.src_addr));
560 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
561 sizeof(ipv6.hdr.dst_addr));
562 memcpy(&ipv6_mask.hdr.dst_addr,
563 &rte_flow_item_ipv6_mask.hdr.dst_addr,
564 sizeof(ipv6.hdr.dst_addr));
567 item.mask = &ipv6_mask;
568 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
569 MLX5_MODIFICATION_TYPE_SET, error);
573 * Convert modify-header set MAC address action to DV specification.
575 * @param[in,out] resource
576 * Pointer to the modify-header resource.
578 * Pointer to action specification.
580 * Pointer to the error structure.
583 * 0 on success, a negative errno value otherwise and rte_errno is set.
586 flow_dv_convert_action_modify_mac
587 (struct mlx5_flow_dv_modify_hdr_resource *resource,
588 const struct rte_flow_action *action,
589 struct rte_flow_error *error)
591 const struct rte_flow_action_set_mac *conf =
592 (const struct rte_flow_action_set_mac *)(action->conf);
593 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
594 struct rte_flow_item_eth eth;
595 struct rte_flow_item_eth eth_mask;
597 memset(ð, 0, sizeof(eth));
598 memset(ð_mask, 0, sizeof(eth_mask));
599 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
600 memcpy(ð.src.addr_bytes, &conf->mac_addr,
601 sizeof(eth.src.addr_bytes));
602 memcpy(ð_mask.src.addr_bytes,
603 &rte_flow_item_eth_mask.src.addr_bytes,
604 sizeof(eth_mask.src.addr_bytes));
606 memcpy(ð.dst.addr_bytes, &conf->mac_addr,
607 sizeof(eth.dst.addr_bytes));
608 memcpy(ð_mask.dst.addr_bytes,
609 &rte_flow_item_eth_mask.dst.addr_bytes,
610 sizeof(eth_mask.dst.addr_bytes));
613 item.mask = ð_mask;
614 return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
615 MLX5_MODIFICATION_TYPE_SET, error);
619 * Convert modify-header set VLAN VID action to DV specification.
621 * @param[in,out] resource
622 * Pointer to the modify-header resource.
624 * Pointer to action specification.
626 * Pointer to the error structure.
629 * 0 on success, a negative errno value otherwise and rte_errno is set.
632 flow_dv_convert_action_modify_vlan_vid
633 (struct mlx5_flow_dv_modify_hdr_resource *resource,
634 const struct rte_flow_action *action,
635 struct rte_flow_error *error)
637 const struct rte_flow_action_of_set_vlan_vid *conf =
638 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
639 int i = resource->actions_num;
640 struct mlx5_modification_cmd *actions = resource->actions;
641 struct field_modify_info *field = modify_vlan_out_first_vid;
643 if (i >= MLX5_MAX_MODIFY_NUM)
644 return rte_flow_error_set(error, EINVAL,
645 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
646 "too many items to modify");
647 actions[i] = (struct mlx5_modification_cmd) {
648 .action_type = MLX5_MODIFICATION_TYPE_SET,
650 .length = field->size,
651 .offset = field->offset,
653 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
654 actions[i].data1 = conf->vlan_vid;
655 actions[i].data1 = actions[i].data1 << 16;
656 resource->actions_num = ++i;
661 * Convert modify-header set TP action to DV specification.
663 * @param[in,out] resource
664 * Pointer to the modify-header resource.
666 * Pointer to action specification.
668 * Pointer to rte_flow_item objects list.
670 * Pointer to flow attributes structure.
671 * @param[in] dev_flow
672 * Pointer to the sub flow.
673 * @param[in] tunnel_decap
674 * Whether action is after tunnel decapsulation.
676 * Pointer to the error structure.
679 * 0 on success, a negative errno value otherwise and rte_errno is set.
682 flow_dv_convert_action_modify_tp
683 (struct mlx5_flow_dv_modify_hdr_resource *resource,
684 const struct rte_flow_action *action,
685 const struct rte_flow_item *items,
686 union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
687 bool tunnel_decap, struct rte_flow_error *error)
689 const struct rte_flow_action_set_tp *conf =
690 (const struct rte_flow_action_set_tp *)(action->conf);
691 struct rte_flow_item item;
692 struct rte_flow_item_udp udp;
693 struct rte_flow_item_udp udp_mask;
694 struct rte_flow_item_tcp tcp;
695 struct rte_flow_item_tcp tcp_mask;
696 struct field_modify_info *field;
699 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
701 memset(&udp, 0, sizeof(udp));
702 memset(&udp_mask, 0, sizeof(udp_mask));
703 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
704 udp.hdr.src_port = conf->port;
705 udp_mask.hdr.src_port =
706 rte_flow_item_udp_mask.hdr.src_port;
708 udp.hdr.dst_port = conf->port;
709 udp_mask.hdr.dst_port =
710 rte_flow_item_udp_mask.hdr.dst_port;
712 item.type = RTE_FLOW_ITEM_TYPE_UDP;
714 item.mask = &udp_mask;
717 MLX5_ASSERT(attr->tcp);
718 memset(&tcp, 0, sizeof(tcp));
719 memset(&tcp_mask, 0, sizeof(tcp_mask));
720 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
721 tcp.hdr.src_port = conf->port;
722 tcp_mask.hdr.src_port =
723 rte_flow_item_tcp_mask.hdr.src_port;
725 tcp.hdr.dst_port = conf->port;
726 tcp_mask.hdr.dst_port =
727 rte_flow_item_tcp_mask.hdr.dst_port;
729 item.type = RTE_FLOW_ITEM_TYPE_TCP;
731 item.mask = &tcp_mask;
734 return flow_dv_convert_modify_action(&item, field, NULL, resource,
735 MLX5_MODIFICATION_TYPE_SET, error);
739 * Convert modify-header set TTL action to DV specification.
741 * @param[in,out] resource
742 * Pointer to the modify-header resource.
744 * Pointer to action specification.
746 * Pointer to rte_flow_item objects list.
748 * Pointer to flow attributes structure.
749 * @param[in] dev_flow
750 * Pointer to the sub flow.
751 * @param[in] tunnel_decap
752 * Whether action is after tunnel decapsulation.
754 * Pointer to the error structure.
757 * 0 on success, a negative errno value otherwise and rte_errno is set.
760 flow_dv_convert_action_modify_ttl
761 (struct mlx5_flow_dv_modify_hdr_resource *resource,
762 const struct rte_flow_action *action,
763 const struct rte_flow_item *items,
764 union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
765 bool tunnel_decap, struct rte_flow_error *error)
767 const struct rte_flow_action_set_ttl *conf =
768 (const struct rte_flow_action_set_ttl *)(action->conf);
769 struct rte_flow_item item;
770 struct rte_flow_item_ipv4 ipv4;
771 struct rte_flow_item_ipv4 ipv4_mask;
772 struct rte_flow_item_ipv6 ipv6;
773 struct rte_flow_item_ipv6 ipv6_mask;
774 struct field_modify_info *field;
777 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
779 memset(&ipv4, 0, sizeof(ipv4));
780 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
781 ipv4.hdr.time_to_live = conf->ttl_value;
782 ipv4_mask.hdr.time_to_live = 0xFF;
783 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
785 item.mask = &ipv4_mask;
788 MLX5_ASSERT(attr->ipv6);
789 memset(&ipv6, 0, sizeof(ipv6));
790 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
791 ipv6.hdr.hop_limits = conf->ttl_value;
792 ipv6_mask.hdr.hop_limits = 0xFF;
793 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
795 item.mask = &ipv6_mask;
798 return flow_dv_convert_modify_action(&item, field, NULL, resource,
799 MLX5_MODIFICATION_TYPE_SET, error);
803 * Convert modify-header decrement TTL action to DV specification.
805 * @param[in,out] resource
806 * Pointer to the modify-header resource.
808 * Pointer to action specification.
810 * Pointer to rte_flow_item objects list.
812 * Pointer to flow attributes structure.
813 * @param[in] dev_flow
814 * Pointer to the sub flow.
815 * @param[in] tunnel_decap
816 * Whether action is after tunnel decapsulation.
818 * Pointer to the error structure.
821 * 0 on success, a negative errno value otherwise and rte_errno is set.
824 flow_dv_convert_action_modify_dec_ttl
825 (struct mlx5_flow_dv_modify_hdr_resource *resource,
826 const struct rte_flow_item *items,
827 union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
828 bool tunnel_decap, struct rte_flow_error *error)
830 struct rte_flow_item item;
831 struct rte_flow_item_ipv4 ipv4;
832 struct rte_flow_item_ipv4 ipv4_mask;
833 struct rte_flow_item_ipv6 ipv6;
834 struct rte_flow_item_ipv6 ipv6_mask;
835 struct field_modify_info *field;
838 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
840 memset(&ipv4, 0, sizeof(ipv4));
841 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
842 ipv4.hdr.time_to_live = 0xFF;
843 ipv4_mask.hdr.time_to_live = 0xFF;
844 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
846 item.mask = &ipv4_mask;
849 MLX5_ASSERT(attr->ipv6);
850 memset(&ipv6, 0, sizeof(ipv6));
851 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
852 ipv6.hdr.hop_limits = 0xFF;
853 ipv6_mask.hdr.hop_limits = 0xFF;
854 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
856 item.mask = &ipv6_mask;
859 return flow_dv_convert_modify_action(&item, field, NULL, resource,
860 MLX5_MODIFICATION_TYPE_ADD, error);
864 * Convert modify-header increment/decrement TCP Sequence number
865 * to DV specification.
867 * @param[in,out] resource
868 * Pointer to the modify-header resource.
870 * Pointer to action specification.
872 * Pointer to the error structure.
875 * 0 on success, a negative errno value otherwise and rte_errno is set.
878 flow_dv_convert_action_modify_tcp_seq
879 (struct mlx5_flow_dv_modify_hdr_resource *resource,
880 const struct rte_flow_action *action,
881 struct rte_flow_error *error)
883 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
884 uint64_t value = rte_be_to_cpu_32(*conf);
885 struct rte_flow_item item;
886 struct rte_flow_item_tcp tcp;
887 struct rte_flow_item_tcp tcp_mask;
889 memset(&tcp, 0, sizeof(tcp));
890 memset(&tcp_mask, 0, sizeof(tcp_mask));
891 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
893 * The HW has no decrement operation, only increment operation.
894 * To simulate decrement X from Y using increment operation
895 * we need to add UINT32_MAX X times to Y.
896 * Each adding of UINT32_MAX decrements Y by 1.
899 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
900 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
901 item.type = RTE_FLOW_ITEM_TYPE_TCP;
903 item.mask = &tcp_mask;
904 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
905 MLX5_MODIFICATION_TYPE_ADD, error);
909 * Convert modify-header increment/decrement TCP Acknowledgment number
910 * to DV specification.
912 * @param[in,out] resource
913 * Pointer to the modify-header resource.
915 * Pointer to action specification.
917 * Pointer to the error structure.
920 * 0 on success, a negative errno value otherwise and rte_errno is set.
923 flow_dv_convert_action_modify_tcp_ack
924 (struct mlx5_flow_dv_modify_hdr_resource *resource,
925 const struct rte_flow_action *action,
926 struct rte_flow_error *error)
928 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
929 uint64_t value = rte_be_to_cpu_32(*conf);
930 struct rte_flow_item item;
931 struct rte_flow_item_tcp tcp;
932 struct rte_flow_item_tcp tcp_mask;
934 memset(&tcp, 0, sizeof(tcp));
935 memset(&tcp_mask, 0, sizeof(tcp_mask));
936 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
938 * The HW has no decrement operation, only increment operation.
939 * To simulate decrement X from Y using increment operation
940 * we need to add UINT32_MAX X times to Y.
941 * Each adding of UINT32_MAX decrements Y by 1.
944 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
945 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
946 item.type = RTE_FLOW_ITEM_TYPE_TCP;
948 item.mask = &tcp_mask;
949 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
950 MLX5_MODIFICATION_TYPE_ADD, error);
953 static enum mlx5_modification_field reg_to_field[] = {
954 [REG_NONE] = MLX5_MODI_OUT_NONE,
955 [REG_A] = MLX5_MODI_META_DATA_REG_A,
956 [REG_B] = MLX5_MODI_META_DATA_REG_B,
957 [REG_C_0] = MLX5_MODI_META_REG_C_0,
958 [REG_C_1] = MLX5_MODI_META_REG_C_1,
959 [REG_C_2] = MLX5_MODI_META_REG_C_2,
960 [REG_C_3] = MLX5_MODI_META_REG_C_3,
961 [REG_C_4] = MLX5_MODI_META_REG_C_4,
962 [REG_C_5] = MLX5_MODI_META_REG_C_5,
963 [REG_C_6] = MLX5_MODI_META_REG_C_6,
964 [REG_C_7] = MLX5_MODI_META_REG_C_7,
968 * Convert register set to DV specification.
970 * @param[in,out] resource
971 * Pointer to the modify-header resource.
973 * Pointer to action specification.
975 * Pointer to the error structure.
978 * 0 on success, a negative errno value otherwise and rte_errno is set.
981 flow_dv_convert_action_set_reg
982 (struct mlx5_flow_dv_modify_hdr_resource *resource,
983 const struct rte_flow_action *action,
984 struct rte_flow_error *error)
986 const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
987 struct mlx5_modification_cmd *actions = resource->actions;
988 uint32_t i = resource->actions_num;
990 if (i >= MLX5_MAX_MODIFY_NUM)
991 return rte_flow_error_set(error, EINVAL,
992 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
993 "too many items to modify");
994 MLX5_ASSERT(conf->id != REG_NONE);
995 MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
996 actions[i] = (struct mlx5_modification_cmd) {
997 .action_type = MLX5_MODIFICATION_TYPE_SET,
998 .field = reg_to_field[conf->id],
1000 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1001 actions[i].data1 = rte_cpu_to_be_32(conf->data);
1003 resource->actions_num = i;
1008 * Convert SET_TAG action to DV specification.
1011 * Pointer to the rte_eth_dev structure.
1012 * @param[in,out] resource
1013 * Pointer to the modify-header resource.
1015 * Pointer to action specification.
1017 * Pointer to the error structure.
1020 * 0 on success, a negative errno value otherwise and rte_errno is set.
1023 flow_dv_convert_action_set_tag
1024 (struct rte_eth_dev *dev,
1025 struct mlx5_flow_dv_modify_hdr_resource *resource,
1026 const struct rte_flow_action_set_tag *conf,
1027 struct rte_flow_error *error)
1029 rte_be32_t data = rte_cpu_to_be_32(conf->data);
1030 rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1031 struct rte_flow_item item = {
1035 struct field_modify_info reg_c_x[] = {
1038 enum mlx5_modification_field reg_type;
1041 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044 MLX5_ASSERT(ret != REG_NONE);
1045 MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1046 reg_type = reg_to_field[ret];
1047 MLX5_ASSERT(reg_type > 0);
1048 reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1049 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1050 MLX5_MODIFICATION_TYPE_SET, error);
1054 * Convert internal COPY_REG action to DV specification.
1057 * Pointer to the rte_eth_dev structure.
1058 * @param[in,out] res
1059 * Pointer to the modify-header resource.
1061 * Pointer to action specification.
1063 * Pointer to the error structure.
1066 * 0 on success, a negative errno value otherwise and rte_errno is set.
1069 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1070 struct mlx5_flow_dv_modify_hdr_resource *res,
1071 const struct rte_flow_action *action,
1072 struct rte_flow_error *error)
1074 const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1075 rte_be32_t mask = RTE_BE32(UINT32_MAX);
1076 struct rte_flow_item item = {
1080 struct field_modify_info reg_src[] = {
1081 {4, 0, reg_to_field[conf->src]},
1084 struct field_modify_info reg_dst = {
1086 .id = reg_to_field[conf->dst],
1088 /* Adjust reg_c[0] usage according to reported mask. */
1089 if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1090 struct mlx5_priv *priv = dev->data->dev_private;
1091 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1093 MLX5_ASSERT(reg_c0);
1094 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1095 if (conf->dst == REG_C_0) {
1096 /* Copy to reg_c[0], within mask only. */
1097 reg_dst.offset = rte_bsf32(reg_c0);
1099 * Mask is ignoring the enianness, because
1100 * there is no conversion in datapath.
1102 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1103 /* Copy from destination lower bits to reg_c[0]. */
1104 mask = reg_c0 >> reg_dst.offset;
1106 /* Copy from destination upper bits to reg_c[0]. */
1107 mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1108 rte_fls_u32(reg_c0));
1111 mask = rte_cpu_to_be_32(reg_c0);
1112 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1113 /* Copy from reg_c[0] to destination lower bits. */
1116 /* Copy from reg_c[0] to destination upper bits. */
1117 reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1118 (rte_fls_u32(reg_c0) -
1123 return flow_dv_convert_modify_action(&item,
1124 reg_src, ®_dst, res,
1125 MLX5_MODIFICATION_TYPE_COPY,
1130 * Convert MARK action to DV specification. This routine is used
1131 * in extensive metadata only and requires metadata register to be
1132 * handled. In legacy mode hardware tag resource is engaged.
1135 * Pointer to the rte_eth_dev structure.
1137 * Pointer to MARK action specification.
1138 * @param[in,out] resource
1139 * Pointer to the modify-header resource.
1141 * Pointer to the error structure.
1144 * 0 on success, a negative errno value otherwise and rte_errno is set.
1147 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1148 const struct rte_flow_action_mark *conf,
1149 struct mlx5_flow_dv_modify_hdr_resource *resource,
1150 struct rte_flow_error *error)
1152 struct mlx5_priv *priv = dev->data->dev_private;
1153 rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1154 priv->sh->dv_mark_mask);
1155 rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1156 struct rte_flow_item item = {
1160 struct field_modify_info reg_c_x[] = {
1161 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1167 return rte_flow_error_set(error, EINVAL,
1168 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1169 NULL, "zero mark action mask");
1170 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1173 MLX5_ASSERT(reg > 0);
1174 if (reg == REG_C_0) {
1175 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1176 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1179 mask = rte_cpu_to_be_32(mask) & msk_c0;
1180 mask = rte_cpu_to_be_32(mask << shl_c0);
1182 reg_c_x[0].id = reg_to_field[reg];
1183 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1184 MLX5_MODIFICATION_TYPE_SET, error);
1188 * Get metadata register index for specified steering domain.
1191 * Pointer to the rte_eth_dev structure.
1193 * Attributes of flow to determine steering domain.
1195 * Pointer to the error structure.
1198 * positive index on success, a negative errno value otherwise
1199 * and rte_errno is set.
1201 static enum modify_reg
1202 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1203 const struct rte_flow_attr *attr,
1204 struct rte_flow_error *error)
1207 mlx5_flow_get_reg_id(dev, attr->transfer ?
1211 MLX5_METADATA_RX, 0, error);
1213 return rte_flow_error_set(error,
1214 ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1215 NULL, "unavailable "
1216 "metadata register");
1221 * Convert SET_META action to DV specification.
1224 * Pointer to the rte_eth_dev structure.
1225 * @param[in,out] resource
1226 * Pointer to the modify-header resource.
1228 * Attributes of flow that includes this item.
1230 * Pointer to action specification.
1232 * Pointer to the error structure.
1235 * 0 on success, a negative errno value otherwise and rte_errno is set.
1238 flow_dv_convert_action_set_meta
1239 (struct rte_eth_dev *dev,
1240 struct mlx5_flow_dv_modify_hdr_resource *resource,
1241 const struct rte_flow_attr *attr,
1242 const struct rte_flow_action_set_meta *conf,
1243 struct rte_flow_error *error)
1245 uint32_t data = conf->data;
1246 uint32_t mask = conf->mask;
1247 struct rte_flow_item item = {
1251 struct field_modify_info reg_c_x[] = {
1254 int reg = flow_dv_get_metadata_reg(dev, attr, error);
1259 * In datapath code there is no endianness
1260 * coversions for perfromance reasons, all
1261 * pattern conversions are done in rte_flow.
1263 if (reg == REG_C_0) {
1264 struct mlx5_priv *priv = dev->data->dev_private;
1265 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1268 MLX5_ASSERT(msk_c0);
1269 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1270 shl_c0 = rte_bsf32(msk_c0);
1272 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1276 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1279 /* The routine expects parameters in memory as big-endian ones. */
1280 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1281 MLX5_MODIFICATION_TYPE_SET, error);
1285 * Convert modify-header set IPv4 DSCP action to DV specification.
1287 * @param[in,out] resource
1288 * Pointer to the modify-header resource.
1290 * Pointer to action specification.
1292 * Pointer to the error structure.
1295 * 0 on success, a negative errno value otherwise and rte_errno is set.
1298 flow_dv_convert_action_modify_ipv4_dscp
1299 (struct mlx5_flow_dv_modify_hdr_resource *resource,
1300 const struct rte_flow_action *action,
1301 struct rte_flow_error *error)
1303 const struct rte_flow_action_set_dscp *conf =
1304 (const struct rte_flow_action_set_dscp *)(action->conf);
1305 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1306 struct rte_flow_item_ipv4 ipv4;
1307 struct rte_flow_item_ipv4 ipv4_mask;
1309 memset(&ipv4, 0, sizeof(ipv4));
1310 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1311 ipv4.hdr.type_of_service = conf->dscp;
1312 ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314 item.mask = &ipv4_mask;
1315 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1316 MLX5_MODIFICATION_TYPE_SET, error);
1320 * Convert modify-header set IPv6 DSCP action to DV specification.
1322 * @param[in,out] resource
1323 * Pointer to the modify-header resource.
1325 * Pointer to action specification.
1327 * Pointer to the error structure.
1330 * 0 on success, a negative errno value otherwise and rte_errno is set.
1333 flow_dv_convert_action_modify_ipv6_dscp
1334 (struct mlx5_flow_dv_modify_hdr_resource *resource,
1335 const struct rte_flow_action *action,
1336 struct rte_flow_error *error)
1338 const struct rte_flow_action_set_dscp *conf =
1339 (const struct rte_flow_action_set_dscp *)(action->conf);
1340 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1341 struct rte_flow_item_ipv6 ipv6;
1342 struct rte_flow_item_ipv6 ipv6_mask;
1344 memset(&ipv6, 0, sizeof(ipv6));
1345 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347 * Even though the DSCP bits offset of IPv6 is not byte aligned,
1348 * rdma-core only accept the DSCP bits byte aligned start from
1349 * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1350 * bits in IPv6 case as rdma-core requires byte aligned value.
1352 ipv6.hdr.vtc_flow = conf->dscp;
1353 ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355 item.mask = &ipv6_mask;
1356 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1357 MLX5_MODIFICATION_TYPE_SET, error);
1361 * Validate MARK item.
1364 * Pointer to the rte_eth_dev structure.
1366 * Item specification.
1368 * Attributes of flow that includes this item.
1370 * Pointer to error structure.
1373 * 0 on success, a negative errno value otherwise and rte_errno is set.
1376 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1377 const struct rte_flow_item *item,
1378 const struct rte_flow_attr *attr __rte_unused,
1379 struct rte_flow_error *error)
1381 struct mlx5_priv *priv = dev->data->dev_private;
1382 struct mlx5_dev_config *config = &priv->config;
1383 const struct rte_flow_item_mark *spec = item->spec;
1384 const struct rte_flow_item_mark *mask = item->mask;
1385 const struct rte_flow_item_mark nic_mask = {
1386 .id = priv->sh->dv_mark_mask,
1390 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1391 return rte_flow_error_set(error, ENOTSUP,
1392 RTE_FLOW_ERROR_TYPE_ITEM, item,
1393 "extended metadata feature"
1395 if (!mlx5_flow_ext_mreg_supported(dev))
1396 return rte_flow_error_set(error, ENOTSUP,
1397 RTE_FLOW_ERROR_TYPE_ITEM, item,
1398 "extended metadata register"
1399 " isn't supported");
1401 return rte_flow_error_set(error, ENOTSUP,
1402 RTE_FLOW_ERROR_TYPE_ITEM, item,
1403 "extended metadata register"
1404 " isn't available");
1405 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1409 return rte_flow_error_set(error, EINVAL,
1410 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412 "data cannot be empty");
1413 if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1414 return rte_flow_error_set(error, EINVAL,
1415 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417 "mark id exceeds the limit");
1421 return rte_flow_error_set(error, EINVAL,
1422 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1423 "mask cannot be zero");
1425 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1426 (const uint8_t *)&nic_mask,
1427 sizeof(struct rte_flow_item_mark),
1435 * Validate META item.
1438 * Pointer to the rte_eth_dev structure.
1440 * Item specification.
1442 * Attributes of flow that includes this item.
1444 * Pointer to error structure.
1447 * 0 on success, a negative errno value otherwise and rte_errno is set.
1450 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1451 const struct rte_flow_item *item,
1452 const struct rte_flow_attr *attr,
1453 struct rte_flow_error *error)
1455 struct mlx5_priv *priv = dev->data->dev_private;
1456 struct mlx5_dev_config *config = &priv->config;
1457 const struct rte_flow_item_meta *spec = item->spec;
1458 const struct rte_flow_item_meta *mask = item->mask;
1459 struct rte_flow_item_meta nic_mask = {
1466 return rte_flow_error_set(error, EINVAL,
1467 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469 "data cannot be empty");
1470 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1471 if (!mlx5_flow_ext_mreg_supported(dev))
1472 return rte_flow_error_set(error, ENOTSUP,
1473 RTE_FLOW_ERROR_TYPE_ITEM, item,
1474 "extended metadata register"
1475 " isn't supported");
1476 reg = flow_dv_get_metadata_reg(dev, attr, error);
1480 return rte_flow_error_set(error, ENOTSUP,
1481 RTE_FLOW_ERROR_TYPE_ITEM, item,
1485 nic_mask.data = priv->sh->dv_meta_mask;
1486 } else if (attr->transfer) {
1487 return rte_flow_error_set(error, ENOTSUP,
1488 RTE_FLOW_ERROR_TYPE_ITEM, item,
1489 "extended metadata feature "
1490 "should be enabled when "
1491 "meta item is requested "
1492 "with e-switch mode ");
1495 mask = &rte_flow_item_meta_mask;
1497 return rte_flow_error_set(error, EINVAL,
1498 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1499 "mask cannot be zero");
1501 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1502 (const uint8_t *)&nic_mask,
1503 sizeof(struct rte_flow_item_meta),
1509 * Validate TAG item.
1512 * Pointer to the rte_eth_dev structure.
1514 * Item specification.
1516 * Attributes of flow that includes this item.
1518 * Pointer to error structure.
1521 * 0 on success, a negative errno value otherwise and rte_errno is set.
1524 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1525 const struct rte_flow_item *item,
1526 const struct rte_flow_attr *attr __rte_unused,
1527 struct rte_flow_error *error)
1529 const struct rte_flow_item_tag *spec = item->spec;
1530 const struct rte_flow_item_tag *mask = item->mask;
1531 const struct rte_flow_item_tag nic_mask = {
1532 .data = RTE_BE32(UINT32_MAX),
1537 if (!mlx5_flow_ext_mreg_supported(dev))
1538 return rte_flow_error_set(error, ENOTSUP,
1539 RTE_FLOW_ERROR_TYPE_ITEM, item,
1540 "extensive metadata register"
1541 " isn't supported");
1543 return rte_flow_error_set(error, EINVAL,
1544 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546 "data cannot be empty");
1548 mask = &rte_flow_item_tag_mask;
1550 return rte_flow_error_set(error, EINVAL,
1551 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1552 "mask cannot be zero");
1554 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1555 (const uint8_t *)&nic_mask,
1556 sizeof(struct rte_flow_item_tag),
1560 if (mask->index != 0xff)
1561 return rte_flow_error_set(error, EINVAL,
1562 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1563 "partial mask for tag index"
1564 " is not supported");
1565 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1568 MLX5_ASSERT(ret != REG_NONE);
1573 * Validate vport item.
1576 * Pointer to the rte_eth_dev structure.
1578 * Item specification.
1580 * Attributes of flow that includes this item.
1581 * @param[in] item_flags
1582 * Bit-fields that holds the items detected until now.
1584 * Pointer to error structure.
1587 * 0 on success, a negative errno value otherwise and rte_errno is set.
1590 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1591 const struct rte_flow_item *item,
1592 const struct rte_flow_attr *attr,
1593 uint64_t item_flags,
1594 struct rte_flow_error *error)
1596 const struct rte_flow_item_port_id *spec = item->spec;
1597 const struct rte_flow_item_port_id *mask = item->mask;
1598 const struct rte_flow_item_port_id switch_mask = {
1601 struct mlx5_priv *esw_priv;
1602 struct mlx5_priv *dev_priv;
1605 if (!attr->transfer)
1606 return rte_flow_error_set(error, EINVAL,
1607 RTE_FLOW_ERROR_TYPE_ITEM,
1609 "match on port id is valid only"
1610 " when transfer flag is enabled");
1611 if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1612 return rte_flow_error_set(error, ENOTSUP,
1613 RTE_FLOW_ERROR_TYPE_ITEM, item,
1614 "multiple source ports are not"
1617 mask = &switch_mask;
1618 if (mask->id != 0xffffffff)
1619 return rte_flow_error_set(error, ENOTSUP,
1620 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622 "no support for partial mask on"
1624 ret = mlx5_flow_item_acceptable
1625 (item, (const uint8_t *)mask,
1626 (const uint8_t *)&rte_flow_item_port_id_mask,
1627 sizeof(struct rte_flow_item_port_id),
1633 esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635 return rte_flow_error_set(error, rte_errno,
1636 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1637 "failed to obtain E-Switch info for"
1639 dev_priv = mlx5_dev_to_eswitch_info(dev);
1641 return rte_flow_error_set(error, rte_errno,
1642 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644 "failed to obtain E-Switch info");
1645 if (esw_priv->domain_id != dev_priv->domain_id)
1646 return rte_flow_error_set(error, EINVAL,
1647 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1648 "cannot match on a port from a"
1649 " different E-Switch");
1654 * Validate VLAN item.
1657 * Item specification.
1658 * @param[in] item_flags
1659 * Bit-fields that holds the items detected until now.
1661 * Ethernet device flow is being created on.
1663 * Pointer to error structure.
1666 * 0 on success, a negative errno value otherwise and rte_errno is set.
1669 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1670 uint64_t item_flags,
1671 struct rte_eth_dev *dev,
1672 struct rte_flow_error *error)
1674 const struct rte_flow_item_vlan *mask = item->mask;
1675 const struct rte_flow_item_vlan nic_mask = {
1676 .tci = RTE_BE16(UINT16_MAX),
1677 .inner_type = RTE_BE16(UINT16_MAX),
1679 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1682 MLX5_FLOW_LAYER_INNER_L4) :
1683 (MLX5_FLOW_LAYER_OUTER_L3 |
1684 MLX5_FLOW_LAYER_OUTER_L4);
1685 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1686 MLX5_FLOW_LAYER_OUTER_VLAN;
1688 if (item_flags & vlanm)
1689 return rte_flow_error_set(error, EINVAL,
1690 RTE_FLOW_ERROR_TYPE_ITEM, item,
1691 "multiple VLAN layers not supported");
1692 else if ((item_flags & l34m) != 0)
1693 return rte_flow_error_set(error, EINVAL,
1694 RTE_FLOW_ERROR_TYPE_ITEM, item,
1695 "VLAN cannot follow L3/L4 layer");
1697 mask = &rte_flow_item_vlan_mask;
1698 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1699 (const uint8_t *)&nic_mask,
1700 sizeof(struct rte_flow_item_vlan),
1704 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1705 struct mlx5_priv *priv = dev->data->dev_private;
1707 if (priv->vmwa_context) {
1709 * Non-NULL context means we have a virtual machine
1710 * and SR-IOV enabled, we have to create VLAN interface
1711 * to make hypervisor to setup E-Switch vport
1712 * context correctly. We avoid creating the multiple
1713 * VLAN interfaces, so we cannot support VLAN tag mask.
1715 return rte_flow_error_set(error, EINVAL,
1716 RTE_FLOW_ERROR_TYPE_ITEM,
1718 "VLAN tag mask is not"
1719 " supported in virtual"
1727 * GTP flags are contained in 1 byte of the format:
1728 * -------------------------------------------
1729 * | bit | 0 - 2 | 3 | 4 | 5 | 6 | 7 |
1730 * |-----------------------------------------|
1731 * | value | Version | PT | Res | E | S | PN |
1732 * -------------------------------------------
1734 * Matching is supported only for GTP flags E, S, PN.
1736 #define MLX5_GTP_FLAGS_MASK 0x07
1739 * Validate GTP item.
1742 * Pointer to the rte_eth_dev structure.
1744 * Item specification.
1745 * @param[in] item_flags
1746 * Bit-fields that holds the items detected until now.
1748 * Pointer to error structure.
1751 * 0 on success, a negative errno value otherwise and rte_errno is set.
1754 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1755 const struct rte_flow_item *item,
1756 uint64_t item_flags,
1757 struct rte_flow_error *error)
1759 struct mlx5_priv *priv = dev->data->dev_private;
1760 const struct rte_flow_item_gtp *spec = item->spec;
1761 const struct rte_flow_item_gtp *mask = item->mask;
1762 const struct rte_flow_item_gtp nic_mask = {
1763 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765 .teid = RTE_BE32(0xffffffff),
1768 if (!priv->config.hca_attr.tunnel_stateless_gtp)
1769 return rte_flow_error_set(error, ENOTSUP,
1770 RTE_FLOW_ERROR_TYPE_ITEM, item,
1771 "GTP support is not enabled");
1772 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1773 return rte_flow_error_set(error, ENOTSUP,
1774 RTE_FLOW_ERROR_TYPE_ITEM, item,
1775 "multiple tunnel layers not"
1777 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1778 return rte_flow_error_set(error, EINVAL,
1779 RTE_FLOW_ERROR_TYPE_ITEM, item,
1780 "no outer UDP layer found");
1782 mask = &rte_flow_item_gtp_mask;
1783 if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1784 return rte_flow_error_set(error, ENOTSUP,
1785 RTE_FLOW_ERROR_TYPE_ITEM, item,
1786 "Match is supported for GTP"
1788 return mlx5_flow_item_acceptable
1789 (item, (const uint8_t *)mask,
1790 (const uint8_t *)&nic_mask,
1791 sizeof(struct rte_flow_item_gtp),
1796 * Validate the pop VLAN action.
1799 * Pointer to the rte_eth_dev structure.
1800 * @param[in] action_flags
1801 * Holds the actions detected until now.
1803 * Pointer to the pop vlan action.
1804 * @param[in] item_flags
1805 * The items found in this flow rule.
1807 * Pointer to flow attributes.
1809 * Pointer to error structure.
1812 * 0 on success, a negative errno value otherwise and rte_errno is set.
1815 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1816 uint64_t action_flags,
1817 const struct rte_flow_action *action,
1818 uint64_t item_flags,
1819 const struct rte_flow_attr *attr,
1820 struct rte_flow_error *error)
1822 const struct mlx5_priv *priv = dev->data->dev_private;
1826 if (!priv->sh->pop_vlan_action)
1827 return rte_flow_error_set(error, ENOTSUP,
1828 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1830 "pop vlan action is not supported");
1832 return rte_flow_error_set(error, ENOTSUP,
1833 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1835 "pop vlan action not supported for "
1837 if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1838 return rte_flow_error_set(error, ENOTSUP,
1839 RTE_FLOW_ERROR_TYPE_ACTION, action,
1840 "no support for multiple VLAN "
1842 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1843 return rte_flow_error_set(error, ENOTSUP,
1844 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1846 "cannot pop vlan without a "
1847 "match on (outer) vlan in the flow");
1848 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1849 return rte_flow_error_set(error, EINVAL,
1850 RTE_FLOW_ERROR_TYPE_ACTION, action,
1851 "wrong action order, port_id should "
1852 "be after pop VLAN action");
1853 if (!attr->transfer && priv->representor)
1854 return rte_flow_error_set(error, ENOTSUP,
1855 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1856 "pop vlan action for VF representor "
1857 "not supported on NIC table");
1862 * Get VLAN default info from vlan match info.
1865 * the list of item specifications.
1867 * pointer VLAN info to fill to.
1870 * 0 on success, a negative errno value otherwise and rte_errno is set.
1873 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1874 struct rte_vlan_hdr *vlan)
1876 const struct rte_flow_item_vlan nic_mask = {
1877 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1878 MLX5DV_FLOW_VLAN_VID_MASK),
1879 .inner_type = RTE_BE16(0xffff),
1884 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1885 int type = items->type;
1887 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1888 type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1891 if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1892 const struct rte_flow_item_vlan *vlan_m = items->mask;
1893 const struct rte_flow_item_vlan *vlan_v = items->spec;
1895 /* If VLAN item in pattern doesn't contain data, return here. */
1900 /* Only full match values are accepted */
1901 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1902 MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1903 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1905 rte_be_to_cpu_16(vlan_v->tci &
1906 MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1908 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1909 MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1910 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1912 rte_be_to_cpu_16(vlan_v->tci &
1913 MLX5DV_FLOW_VLAN_VID_MASK_BE);
1915 if (vlan_m->inner_type == nic_mask.inner_type)
1916 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1917 vlan_m->inner_type);
1922 * Validate the push VLAN action.
1925 * Pointer to the rte_eth_dev structure.
1926 * @param[in] action_flags
1927 * Holds the actions detected until now.
1928 * @param[in] item_flags
1929 * The items found in this flow rule.
1931 * Pointer to the action structure.
1933 * Pointer to flow attributes
1935 * Pointer to error structure.
1938 * 0 on success, a negative errno value otherwise and rte_errno is set.
1941 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1942 uint64_t action_flags,
1943 const struct rte_flow_item_vlan *vlan_m,
1944 const struct rte_flow_action *action,
1945 const struct rte_flow_attr *attr,
1946 struct rte_flow_error *error)
1948 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1949 const struct mlx5_priv *priv = dev->data->dev_private;
1951 if (!attr->transfer && attr->ingress)
1952 return rte_flow_error_set(error, ENOTSUP,
1953 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1955 "push VLAN action not supported for "
1957 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1958 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1959 return rte_flow_error_set(error, EINVAL,
1960 RTE_FLOW_ERROR_TYPE_ACTION, action,
1961 "invalid vlan ethertype");
1962 if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1963 return rte_flow_error_set(error, ENOTSUP,
1964 RTE_FLOW_ERROR_TYPE_ACTION, action,
1965 "no support for multiple VLAN "
1967 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1968 return rte_flow_error_set(error, EINVAL,
1969 RTE_FLOW_ERROR_TYPE_ACTION, action,
1970 "wrong action order, port_id should "
1971 "be after push VLAN");
1972 if (!attr->transfer && priv->representor)
1973 return rte_flow_error_set(error, ENOTSUP,
1974 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1975 "push vlan action for VF representor "
1976 "not supported on NIC table");
1978 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1979 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1980 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1981 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1982 !(mlx5_flow_find_action
1983 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1984 return rte_flow_error_set(error, EINVAL,
1985 RTE_FLOW_ERROR_TYPE_ACTION, action,
1986 "not full match mask on VLAN PCP and "
1987 "there is no of_set_vlan_pcp action, "
1988 "push VLAN action cannot figure out "
1991 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1992 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1993 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1994 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1995 !(mlx5_flow_find_action
1996 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1997 return rte_flow_error_set(error, EINVAL,
1998 RTE_FLOW_ERROR_TYPE_ACTION, action,
1999 "not full match mask on VLAN VID and "
2000 "there is no of_set_vlan_vid action, "
2001 "push VLAN action cannot figure out "
2008 * Validate the set VLAN PCP.
2010 * @param[in] action_flags
2011 * Holds the actions detected until now.
2012 * @param[in] actions
2013 * Pointer to the list of actions remaining in the flow rule.
2015 * Pointer to error structure.
2018 * 0 on success, a negative errno value otherwise and rte_errno is set.
2021 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2022 const struct rte_flow_action actions[],
2023 struct rte_flow_error *error)
2025 const struct rte_flow_action *action = actions;
2026 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2028 if (conf->vlan_pcp > 7)
2029 return rte_flow_error_set(error, EINVAL,
2030 RTE_FLOW_ERROR_TYPE_ACTION, action,
2031 "VLAN PCP value is too big");
2032 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2033 return rte_flow_error_set(error, ENOTSUP,
2034 RTE_FLOW_ERROR_TYPE_ACTION, action,
2035 "set VLAN PCP action must follow "
2036 "the push VLAN action");
2037 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2038 return rte_flow_error_set(error, ENOTSUP,
2039 RTE_FLOW_ERROR_TYPE_ACTION, action,
2040 "Multiple VLAN PCP modification are "
2042 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2043 return rte_flow_error_set(error, EINVAL,
2044 RTE_FLOW_ERROR_TYPE_ACTION, action,
2045 "wrong action order, port_id should "
2046 "be after set VLAN PCP");
2051 * Validate the set VLAN VID.
2053 * @param[in] item_flags
2054 * Holds the items detected in this rule.
2055 * @param[in] action_flags
2056 * Holds the actions detected until now.
2057 * @param[in] actions
2058 * Pointer to the list of actions remaining in the flow rule.
2060 * Pointer to error structure.
2063 * 0 on success, a negative errno value otherwise and rte_errno is set.
2066 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2067 uint64_t action_flags,
2068 const struct rte_flow_action actions[],
2069 struct rte_flow_error *error)
2071 const struct rte_flow_action *action = actions;
2072 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2074 if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2075 return rte_flow_error_set(error, EINVAL,
2076 RTE_FLOW_ERROR_TYPE_ACTION, action,
2077 "VLAN VID value is too big");
2078 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2079 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2080 return rte_flow_error_set(error, ENOTSUP,
2081 RTE_FLOW_ERROR_TYPE_ACTION, action,
2082 "set VLAN VID action must follow push"
2083 " VLAN action or match on VLAN item");
2084 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2085 return rte_flow_error_set(error, ENOTSUP,
2086 RTE_FLOW_ERROR_TYPE_ACTION, action,
2087 "Multiple VLAN VID modifications are "
2089 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2090 return rte_flow_error_set(error, EINVAL,
2091 RTE_FLOW_ERROR_TYPE_ACTION, action,
2092 "wrong action order, port_id should "
2093 "be after set VLAN VID");
2098 * Validate the FLAG action.
2101 * Pointer to the rte_eth_dev structure.
2102 * @param[in] action_flags
2103 * Holds the actions detected until now.
2105 * Pointer to flow attributes
2107 * Pointer to error structure.
2110 * 0 on success, a negative errno value otherwise and rte_errno is set.
2113 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2114 uint64_t action_flags,
2115 const struct rte_flow_attr *attr,
2116 struct rte_flow_error *error)
2118 struct mlx5_priv *priv = dev->data->dev_private;
2119 struct mlx5_dev_config *config = &priv->config;
2122 /* Fall back if no extended metadata register support. */
2123 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2124 return mlx5_flow_validate_action_flag(action_flags, attr,
2126 /* Extensive metadata mode requires registers. */
2127 if (!mlx5_flow_ext_mreg_supported(dev))
2128 return rte_flow_error_set(error, ENOTSUP,
2129 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2130 "no metadata registers "
2131 "to support flag action");
2132 if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2133 return rte_flow_error_set(error, ENOTSUP,
2134 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2135 "extended metadata register"
2136 " isn't available");
2137 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2140 MLX5_ASSERT(ret > 0);
2141 if (action_flags & MLX5_FLOW_ACTION_MARK)
2142 return rte_flow_error_set(error, EINVAL,
2143 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2144 "can't mark and flag in same flow");
2145 if (action_flags & MLX5_FLOW_ACTION_FLAG)
2146 return rte_flow_error_set(error, EINVAL,
2147 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2149 " actions in same flow");
2154 * Validate MARK action.
2157 * Pointer to the rte_eth_dev structure.
2159 * Pointer to action.
2160 * @param[in] action_flags
2161 * Holds the actions detected until now.
2163 * Pointer to flow attributes
2165 * Pointer to error structure.
2168 * 0 on success, a negative errno value otherwise and rte_errno is set.
2171 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2172 const struct rte_flow_action *action,
2173 uint64_t action_flags,
2174 const struct rte_flow_attr *attr,
2175 struct rte_flow_error *error)
2177 struct mlx5_priv *priv = dev->data->dev_private;
2178 struct mlx5_dev_config *config = &priv->config;
2179 const struct rte_flow_action_mark *mark = action->conf;
2182 /* Fall back if no extended metadata register support. */
2183 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2184 return mlx5_flow_validate_action_mark(action, action_flags,
2186 /* Extensive metadata mode requires registers. */
2187 if (!mlx5_flow_ext_mreg_supported(dev))
2188 return rte_flow_error_set(error, ENOTSUP,
2189 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2190 "no metadata registers "
2191 "to support mark action");
2192 if (!priv->sh->dv_mark_mask)
2193 return rte_flow_error_set(error, ENOTSUP,
2194 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2195 "extended metadata register"
2196 " isn't available");
2197 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2200 MLX5_ASSERT(ret > 0);
2202 return rte_flow_error_set(error, EINVAL,
2203 RTE_FLOW_ERROR_TYPE_ACTION, action,
2204 "configuration cannot be null");
2205 if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2206 return rte_flow_error_set(error, EINVAL,
2207 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2209 "mark id exceeds the limit");
2210 if (action_flags & MLX5_FLOW_ACTION_FLAG)
2211 return rte_flow_error_set(error, EINVAL,
2212 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2213 "can't flag and mark in same flow");
2214 if (action_flags & MLX5_FLOW_ACTION_MARK)
2215 return rte_flow_error_set(error, EINVAL,
2216 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2217 "can't have 2 mark actions in same"
2223 * Validate SET_META action.
2226 * Pointer to the rte_eth_dev structure.
2228 * Pointer to the action structure.
2229 * @param[in] action_flags
2230 * Holds the actions detected until now.
2232 * Pointer to flow attributes
2234 * Pointer to error structure.
2237 * 0 on success, a negative errno value otherwise and rte_errno is set.
2240 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2241 const struct rte_flow_action *action,
2242 uint64_t action_flags __rte_unused,
2243 const struct rte_flow_attr *attr,
2244 struct rte_flow_error *error)
2246 const struct rte_flow_action_set_meta *conf;
2247 uint32_t nic_mask = UINT32_MAX;
2250 if (!mlx5_flow_ext_mreg_supported(dev))
2251 return rte_flow_error_set(error, ENOTSUP,
2252 RTE_FLOW_ERROR_TYPE_ACTION, action,
2253 "extended metadata register"
2254 " isn't supported");
2255 reg = flow_dv_get_metadata_reg(dev, attr, error);
2258 if (reg != REG_A && reg != REG_B) {
2259 struct mlx5_priv *priv = dev->data->dev_private;
2261 nic_mask = priv->sh->dv_meta_mask;
2263 if (!(action->conf))
2264 return rte_flow_error_set(error, EINVAL,
2265 RTE_FLOW_ERROR_TYPE_ACTION, action,
2266 "configuration cannot be null");
2267 conf = (const struct rte_flow_action_set_meta *)action->conf;
2269 return rte_flow_error_set(error, EINVAL,
2270 RTE_FLOW_ERROR_TYPE_ACTION, action,
2271 "zero mask doesn't have any effect");
2272 if (conf->mask & ~nic_mask)
2273 return rte_flow_error_set(error, EINVAL,
2274 RTE_FLOW_ERROR_TYPE_ACTION, action,
2275 "meta data must be within reg C0");
2280 * Validate SET_TAG action.
2283 * Pointer to the rte_eth_dev structure.
2285 * Pointer to the action structure.
2286 * @param[in] action_flags
2287 * Holds the actions detected until now.
2289 * Pointer to flow attributes
2291 * Pointer to error structure.
2294 * 0 on success, a negative errno value otherwise and rte_errno is set.
2297 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2298 const struct rte_flow_action *action,
2299 uint64_t action_flags,
2300 const struct rte_flow_attr *attr,
2301 struct rte_flow_error *error)
2303 const struct rte_flow_action_set_tag *conf;
2304 const uint64_t terminal_action_flags =
2305 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2306 MLX5_FLOW_ACTION_RSS;
2309 if (!mlx5_flow_ext_mreg_supported(dev))
2310 return rte_flow_error_set(error, ENOTSUP,
2311 RTE_FLOW_ERROR_TYPE_ACTION, action,
2312 "extensive metadata register"
2313 " isn't supported");
2314 if (!(action->conf))
2315 return rte_flow_error_set(error, EINVAL,
2316 RTE_FLOW_ERROR_TYPE_ACTION, action,
2317 "configuration cannot be null");
2318 conf = (const struct rte_flow_action_set_tag *)action->conf;
2320 return rte_flow_error_set(error, EINVAL,
2321 RTE_FLOW_ERROR_TYPE_ACTION, action,
2322 "zero mask doesn't have any effect");
2323 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2326 if (!attr->transfer && attr->ingress &&
2327 (action_flags & terminal_action_flags))
2328 return rte_flow_error_set(error, EINVAL,
2329 RTE_FLOW_ERROR_TYPE_ACTION, action,
2330 "set_tag has no effect"
2331 " with terminal actions");
2336 * Validate count action.
2339 * Pointer to rte_eth_dev structure.
2341 * Pointer to error structure.
2344 * 0 on success, a negative errno value otherwise and rte_errno is set.
2347 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2348 struct rte_flow_error *error)
2350 struct mlx5_priv *priv = dev->data->dev_private;
2352 if (!priv->config.devx)
2354 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2358 return rte_flow_error_set
2360 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2362 "count action not supported");
2366 * Validate the L2 encap action.
2369 * Pointer to the rte_eth_dev structure.
2370 * @param[in] action_flags
2371 * Holds the actions detected until now.
2373 * Pointer to the action structure.
2375 * Pointer to flow attributes.
2377 * Pointer to error structure.
2380 * 0 on success, a negative errno value otherwise and rte_errno is set.
2383 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2384 uint64_t action_flags,
2385 const struct rte_flow_action *action,
2386 const struct rte_flow_attr *attr,
2387 struct rte_flow_error *error)
2389 const struct mlx5_priv *priv = dev->data->dev_private;
2391 if (!(action->conf))
2392 return rte_flow_error_set(error, EINVAL,
2393 RTE_FLOW_ERROR_TYPE_ACTION, action,
2394 "configuration cannot be null");
2395 if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2396 return rte_flow_error_set(error, EINVAL,
2397 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2398 "can only have a single encap action "
2400 if (!attr->transfer && priv->representor)
2401 return rte_flow_error_set(error, ENOTSUP,
2402 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2403 "encap action for VF representor "
2404 "not supported on NIC table");
2409 * Validate a decap action.
2412 * Pointer to the rte_eth_dev structure.
2413 * @param[in] action_flags
2414 * Holds the actions detected until now.
2416 * Pointer to flow attributes
2418 * Pointer to error structure.
2421 * 0 on success, a negative errno value otherwise and rte_errno is set.
2424 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2425 uint64_t action_flags,
2426 const struct rte_flow_attr *attr,
2427 struct rte_flow_error *error)
2429 const struct mlx5_priv *priv = dev->data->dev_private;
2431 if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2432 return rte_flow_error_set(error, ENOTSUP,
2433 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2435 MLX5_FLOW_ACTION_DECAP ? "can only "
2436 "have a single decap action" : "decap "
2437 "after encap is not supported");
2438 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2439 return rte_flow_error_set(error, EINVAL,
2440 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2441 "can't have decap action after"
2444 return rte_flow_error_set(error, ENOTSUP,
2445 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2447 "decap action not supported for "
2449 if (!attr->transfer && priv->representor)
2450 return rte_flow_error_set(error, ENOTSUP,
2451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2452 "decap action for VF representor "
2453 "not supported on NIC table");
2457 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2460 * Validate the raw encap and decap actions.
2463 * Pointer to the rte_eth_dev structure.
2465 * Pointer to the decap action.
2467 * Pointer to the encap action.
2469 * Pointer to flow attributes
2470 * @param[in/out] action_flags
2471 * Holds the actions detected until now.
2472 * @param[out] actions_n
2473 * pointer to the number of actions counter.
2475 * Pointer to error structure.
2478 * 0 on success, a negative errno value otherwise and rte_errno is set.
2481 flow_dv_validate_action_raw_encap_decap
2482 (struct rte_eth_dev *dev,
2483 const struct rte_flow_action_raw_decap *decap,
2484 const struct rte_flow_action_raw_encap *encap,
2485 const struct rte_flow_attr *attr, uint64_t *action_flags,
2486 int *actions_n, struct rte_flow_error *error)
2488 const struct mlx5_priv *priv = dev->data->dev_private;
2491 if (encap && (!encap->size || !encap->data))
2492 return rte_flow_error_set(error, EINVAL,
2493 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2494 "raw encap data cannot be empty");
2495 if (decap && encap) {
2496 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2497 encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2500 else if (encap->size <=
2501 MLX5_ENCAPSULATION_DECISION_SIZE &&
2503 MLX5_ENCAPSULATION_DECISION_SIZE)
2506 else if (encap->size >
2507 MLX5_ENCAPSULATION_DECISION_SIZE &&
2509 MLX5_ENCAPSULATION_DECISION_SIZE)
2510 /* 2 L2 actions: encap and decap. */
2513 return rte_flow_error_set(error,
2515 RTE_FLOW_ERROR_TYPE_ACTION,
2516 NULL, "unsupported too small "
2517 "raw decap and too small raw "
2518 "encap combination");
2521 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2525 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2529 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2530 return rte_flow_error_set(error, ENOTSUP,
2531 RTE_FLOW_ERROR_TYPE_ACTION,
2533 "small raw encap size");
2534 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2535 return rte_flow_error_set(error, EINVAL,
2536 RTE_FLOW_ERROR_TYPE_ACTION,
2538 "more than one encap action");
2539 if (!attr->transfer && priv->representor)
2540 return rte_flow_error_set
2542 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2543 "encap action for VF representor "
2544 "not supported on NIC table");
2545 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2552 * Find existing encap/decap resource or create and register a new one.
2554 * @param[in, out] dev
2555 * Pointer to rte_eth_dev structure.
2556 * @param[in, out] resource
2557 * Pointer to encap/decap resource.
2558 * @parm[in, out] dev_flow
2559 * Pointer to the dev_flow.
2561 * pointer to error structure.
2564 * 0 on success otherwise -errno and errno is set.
2567 flow_dv_encap_decap_resource_register
2568 (struct rte_eth_dev *dev,
2569 struct mlx5_flow_dv_encap_decap_resource *resource,
2570 struct mlx5_flow *dev_flow,
2571 struct rte_flow_error *error)
2573 struct mlx5_priv *priv = dev->data->dev_private;
2574 struct mlx5_dev_ctx_shared *sh = priv->sh;
2575 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2576 struct mlx5dv_dr_domain *domain;
2580 resource->flags = dev_flow->dv.group ? 0 : 1;
2581 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2582 domain = sh->fdb_domain;
2583 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2584 domain = sh->rx_domain;
2586 domain = sh->tx_domain;
2587 /* Lookup a matching resource from cache. */
2588 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2589 cache_resource, next) {
2590 if (resource->reformat_type == cache_resource->reformat_type &&
2591 resource->ft_type == cache_resource->ft_type &&
2592 resource->flags == cache_resource->flags &&
2593 resource->size == cache_resource->size &&
2594 !memcmp((const void *)resource->buf,
2595 (const void *)cache_resource->buf,
2597 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2598 (void *)cache_resource,
2599 rte_atomic32_read(&cache_resource->refcnt));
2600 rte_atomic32_inc(&cache_resource->refcnt);
2601 dev_flow->handle->dvh.rix_encap_decap = idx;
2602 dev_flow->dv.encap_decap = cache_resource;
2606 /* Register new encap/decap resource. */
2607 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2608 &dev_flow->handle->dvh.rix_encap_decap);
2609 if (!cache_resource)
2610 return rte_flow_error_set(error, ENOMEM,
2611 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2612 "cannot allocate resource memory");
2613 *cache_resource = *resource;
2614 ret = mlx5_flow_os_create_flow_action_packet_reformat
2615 (sh->ctx, domain, cache_resource,
2616 &cache_resource->action);
2618 rte_free(cache_resource);
2619 return rte_flow_error_set(error, ENOMEM,
2620 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2621 NULL, "cannot create action");
2623 rte_atomic32_init(&cache_resource->refcnt);
2624 rte_atomic32_inc(&cache_resource->refcnt);
2625 ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2626 dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2628 dev_flow->dv.encap_decap = cache_resource;
2629 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2630 (void *)cache_resource,
2631 rte_atomic32_read(&cache_resource->refcnt));
2636 * Find existing table jump resource or create and register a new one.
2638 * @param[in, out] dev
2639 * Pointer to rte_eth_dev structure.
2640 * @param[in, out] tbl
2641 * Pointer to flow table resource.
2642 * @parm[in, out] dev_flow
2643 * Pointer to the dev_flow.
2645 * pointer to error structure.
2648 * 0 on success otherwise -errno and errno is set.
2651 flow_dv_jump_tbl_resource_register
2652 (struct rte_eth_dev *dev __rte_unused,
2653 struct mlx5_flow_tbl_resource *tbl,
2654 struct mlx5_flow *dev_flow,
2655 struct rte_flow_error *error)
2657 struct mlx5_flow_tbl_data_entry *tbl_data =
2658 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2662 cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2664 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2665 (tbl->obj, &tbl_data->jump.action);
2667 return rte_flow_error_set(error, ENOMEM,
2668 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2669 NULL, "cannot create jump action");
2670 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2671 (void *)&tbl_data->jump, cnt);
2673 /* old jump should not make the table ref++. */
2674 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2675 MLX5_ASSERT(tbl_data->jump.action);
2676 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2677 (void *)&tbl_data->jump, cnt);
2679 rte_atomic32_inc(&tbl_data->jump.refcnt);
2680 dev_flow->handle->rix_jump = tbl_data->idx;
2681 dev_flow->dv.jump = &tbl_data->jump;
2686 * Find existing default miss resource or create and register a new one.
2688 * @param[in, out] dev
2689 * Pointer to rte_eth_dev structure.
2691 * pointer to error structure.
2694 * 0 on success otherwise -errno and errno is set.
2697 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2698 struct rte_flow_error *error)
2700 struct mlx5_priv *priv = dev->data->dev_private;
2701 struct mlx5_dev_ctx_shared *sh = priv->sh;
2702 struct mlx5_flow_default_miss_resource *cache_resource =
2704 int cnt = rte_atomic32_read(&cache_resource->refcnt);
2707 MLX5_ASSERT(cache_resource->action);
2708 cache_resource->action =
2709 mlx5_glue->dr_create_flow_action_default_miss();
2710 if (!cache_resource->action)
2711 return rte_flow_error_set(error, ENOMEM,
2712 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2713 "cannot create default miss action");
2714 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2715 (void *)cache_resource->action, cnt);
2717 rte_atomic32_inc(&cache_resource->refcnt);
2722 * Find existing table port ID resource or create and register a new one.
2724 * @param[in, out] dev
2725 * Pointer to rte_eth_dev structure.
2726 * @param[in, out] resource
2727 * Pointer to port ID action resource.
2728 * @parm[in, out] dev_flow
2729 * Pointer to the dev_flow.
2731 * pointer to error structure.
2734 * 0 on success otherwise -errno and errno is set.
2737 flow_dv_port_id_action_resource_register
2738 (struct rte_eth_dev *dev,
2739 struct mlx5_flow_dv_port_id_action_resource *resource,
2740 struct mlx5_flow *dev_flow,
2741 struct rte_flow_error *error)
2743 struct mlx5_priv *priv = dev->data->dev_private;
2744 struct mlx5_dev_ctx_shared *sh = priv->sh;
2745 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2749 /* Lookup a matching resource from cache. */
2750 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2751 idx, cache_resource, next) {
2752 if (resource->port_id == cache_resource->port_id) {
2753 DRV_LOG(DEBUG, "port id action resource resource %p: "
2755 (void *)cache_resource,
2756 rte_atomic32_read(&cache_resource->refcnt));
2757 rte_atomic32_inc(&cache_resource->refcnt);
2758 dev_flow->handle->rix_port_id_action = idx;
2759 dev_flow->dv.port_id_action = cache_resource;
2763 /* Register new port id action resource. */
2764 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2765 &dev_flow->handle->rix_port_id_action);
2766 if (!cache_resource)
2767 return rte_flow_error_set(error, ENOMEM,
2768 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2769 "cannot allocate resource memory");
2770 *cache_resource = *resource;
2771 ret = mlx5_flow_os_create_flow_action_dest_port
2772 (priv->sh->fdb_domain, resource->port_id,
2773 &cache_resource->action);
2775 rte_free(cache_resource);
2776 return rte_flow_error_set(error, ENOMEM,
2777 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2778 NULL, "cannot create action");
2780 rte_atomic32_init(&cache_resource->refcnt);
2781 rte_atomic32_inc(&cache_resource->refcnt);
2782 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2783 dev_flow->handle->rix_port_id_action, cache_resource,
2785 dev_flow->dv.port_id_action = cache_resource;
2786 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2787 (void *)cache_resource,
2788 rte_atomic32_read(&cache_resource->refcnt));
2793 * Find existing push vlan resource or create and register a new one.
2795 * @param [in, out] dev
2796 * Pointer to rte_eth_dev structure.
2797 * @param[in, out] resource
2798 * Pointer to port ID action resource.
2799 * @parm[in, out] dev_flow
2800 * Pointer to the dev_flow.
2802 * pointer to error structure.
2805 * 0 on success otherwise -errno and errno is set.
2808 flow_dv_push_vlan_action_resource_register
2809 (struct rte_eth_dev *dev,
2810 struct mlx5_flow_dv_push_vlan_action_resource *resource,
2811 struct mlx5_flow *dev_flow,
2812 struct rte_flow_error *error)
2814 struct mlx5_priv *priv = dev->data->dev_private;
2815 struct mlx5_dev_ctx_shared *sh = priv->sh;
2816 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2817 struct mlx5dv_dr_domain *domain;
2821 /* Lookup a matching resource from cache. */
2822 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2823 sh->push_vlan_action_list, idx, cache_resource, next) {
2824 if (resource->vlan_tag == cache_resource->vlan_tag &&
2825 resource->ft_type == cache_resource->ft_type) {
2826 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2828 (void *)cache_resource,
2829 rte_atomic32_read(&cache_resource->refcnt));
2830 rte_atomic32_inc(&cache_resource->refcnt);
2831 dev_flow->handle->dvh.rix_push_vlan = idx;
2832 dev_flow->dv.push_vlan_res = cache_resource;
2836 /* Register new push_vlan action resource. */
2837 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2838 &dev_flow->handle->dvh.rix_push_vlan);
2839 if (!cache_resource)
2840 return rte_flow_error_set(error, ENOMEM,
2841 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2842 "cannot allocate resource memory");
2843 *cache_resource = *resource;
2844 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2845 domain = sh->fdb_domain;
2846 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2847 domain = sh->rx_domain;
2849 domain = sh->tx_domain;
2850 ret = mlx5_flow_os_create_flow_action_push_vlan
2851 (domain, resource->vlan_tag,
2852 &cache_resource->action);
2854 rte_free(cache_resource);
2855 return rte_flow_error_set(error, ENOMEM,
2856 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2857 NULL, "cannot create action");
2859 rte_atomic32_init(&cache_resource->refcnt);
2860 rte_atomic32_inc(&cache_resource->refcnt);
2861 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2862 &sh->push_vlan_action_list,
2863 dev_flow->handle->dvh.rix_push_vlan,
2864 cache_resource, next);
2865 dev_flow->dv.push_vlan_res = cache_resource;
2866 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2867 (void *)cache_resource,
2868 rte_atomic32_read(&cache_resource->refcnt));
2872 * Get the size of specific rte_flow_item_type
2874 * @param[in] item_type
2875 * Tested rte_flow_item_type.
2878 * sizeof struct item_type, 0 if void or irrelevant.
2881 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2885 switch (item_type) {
2886 case RTE_FLOW_ITEM_TYPE_ETH:
2887 retval = sizeof(struct rte_flow_item_eth);
2889 case RTE_FLOW_ITEM_TYPE_VLAN:
2890 retval = sizeof(struct rte_flow_item_vlan);
2892 case RTE_FLOW_ITEM_TYPE_IPV4:
2893 retval = sizeof(struct rte_flow_item_ipv4);
2895 case RTE_FLOW_ITEM_TYPE_IPV6:
2896 retval = sizeof(struct rte_flow_item_ipv6);
2898 case RTE_FLOW_ITEM_TYPE_UDP:
2899 retval = sizeof(struct rte_flow_item_udp);
2901 case RTE_FLOW_ITEM_TYPE_TCP:
2902 retval = sizeof(struct rte_flow_item_tcp);
2904 case RTE_FLOW_ITEM_TYPE_VXLAN:
2905 retval = sizeof(struct rte_flow_item_vxlan);
2907 case RTE_FLOW_ITEM_TYPE_GRE:
2908 retval = sizeof(struct rte_flow_item_gre);
2910 case RTE_FLOW_ITEM_TYPE_NVGRE:
2911 retval = sizeof(struct rte_flow_item_nvgre);
2913 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2914 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2916 case RTE_FLOW_ITEM_TYPE_MPLS:
2917 retval = sizeof(struct rte_flow_item_mpls);
2919 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2927 #define MLX5_ENCAP_IPV4_VERSION 0x40
2928 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
2929 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
2930 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
2931 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
2932 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
2933 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
2936 * Convert the encap action data from list of rte_flow_item to raw buffer
2939 * Pointer to rte_flow_item objects list.
2941 * Pointer to the output buffer.
2943 * Pointer to the output buffer size.
2945 * Pointer to the error structure.
2948 * 0 on success, a negative errno value otherwise and rte_errno is set.
2951 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2952 size_t *size, struct rte_flow_error *error)
2954 struct rte_ether_hdr *eth = NULL;
2955 struct rte_vlan_hdr *vlan = NULL;
2956 struct rte_ipv4_hdr *ipv4 = NULL;
2957 struct rte_ipv6_hdr *ipv6 = NULL;
2958 struct rte_udp_hdr *udp = NULL;
2959 struct rte_vxlan_hdr *vxlan = NULL;
2960 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2961 struct rte_gre_hdr *gre = NULL;
2963 size_t temp_size = 0;
2966 return rte_flow_error_set(error, EINVAL,
2967 RTE_FLOW_ERROR_TYPE_ACTION,
2968 NULL, "invalid empty data");
2969 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2970 len = flow_dv_get_item_len(items->type);
2971 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2972 return rte_flow_error_set(error, EINVAL,
2973 RTE_FLOW_ERROR_TYPE_ACTION,
2974 (void *)items->type,
2975 "items total size is too big"
2976 " for encap action");
2977 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2978 switch (items->type) {
2979 case RTE_FLOW_ITEM_TYPE_ETH:
2980 eth = (struct rte_ether_hdr *)&buf[temp_size];
2982 case RTE_FLOW_ITEM_TYPE_VLAN:
2983 vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2985 return rte_flow_error_set(error, EINVAL,
2986 RTE_FLOW_ERROR_TYPE_ACTION,
2987 (void *)items->type,
2988 "eth header not found");
2989 if (!eth->ether_type)
2990 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2992 case RTE_FLOW_ITEM_TYPE_IPV4:
2993 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
2995 return rte_flow_error_set(error, EINVAL,
2996 RTE_FLOW_ERROR_TYPE_ACTION,
2997 (void *)items->type,
2998 "neither eth nor vlan"
3000 if (vlan && !vlan->eth_proto)
3001 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3002 else if (eth && !eth->ether_type)
3003 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3004 if (!ipv4->version_ihl)
3005 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3006 MLX5_ENCAP_IPV4_IHL_MIN;
3007 if (!ipv4->time_to_live)
3008 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3010 case RTE_FLOW_ITEM_TYPE_IPV6:
3011 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3013 return rte_flow_error_set(error, EINVAL,
3014 RTE_FLOW_ERROR_TYPE_ACTION,
3015 (void *)items->type,
3016 "neither eth nor vlan"
3018 if (vlan && !vlan->eth_proto)
3019 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3020 else if (eth && !eth->ether_type)
3021 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3022 if (!ipv6->vtc_flow)
3024 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3025 if (!ipv6->hop_limits)
3026 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3028 case RTE_FLOW_ITEM_TYPE_UDP:
3029 udp = (struct rte_udp_hdr *)&buf[temp_size];
3031 return rte_flow_error_set(error, EINVAL,
3032 RTE_FLOW_ERROR_TYPE_ACTION,
3033 (void *)items->type,
3034 "ip header not found");
3035 if (ipv4 && !ipv4->next_proto_id)
3036 ipv4->next_proto_id = IPPROTO_UDP;
3037 else if (ipv6 && !ipv6->proto)
3038 ipv6->proto = IPPROTO_UDP;
3040 case RTE_FLOW_ITEM_TYPE_VXLAN:
3041 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3043 return rte_flow_error_set(error, EINVAL,
3044 RTE_FLOW_ERROR_TYPE_ACTION,
3045 (void *)items->type,
3046 "udp header not found");
3048 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3049 if (!vxlan->vx_flags)
3051 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3053 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3054 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3056 return rte_flow_error_set(error, EINVAL,
3057 RTE_FLOW_ERROR_TYPE_ACTION,
3058 (void *)items->type,
3059 "udp header not found");
3060 if (!vxlan_gpe->proto)
3061 return rte_flow_error_set(error, EINVAL,
3062 RTE_FLOW_ERROR_TYPE_ACTION,
3063 (void *)items->type,
3064 "next protocol not found");
3067 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3068 if (!vxlan_gpe->vx_flags)
3069 vxlan_gpe->vx_flags =
3070 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3072 case RTE_FLOW_ITEM_TYPE_GRE:
3073 case RTE_FLOW_ITEM_TYPE_NVGRE:
3074 gre = (struct rte_gre_hdr *)&buf[temp_size];
3076 return rte_flow_error_set(error, EINVAL,
3077 RTE_FLOW_ERROR_TYPE_ACTION,
3078 (void *)items->type,
3079 "next protocol not found");
3081 return rte_flow_error_set(error, EINVAL,
3082 RTE_FLOW_ERROR_TYPE_ACTION,
3083 (void *)items->type,
3084 "ip header not found");
3085 if (ipv4 && !ipv4->next_proto_id)
3086 ipv4->next_proto_id = IPPROTO_GRE;
3087 else if (ipv6 && !ipv6->proto)
3088 ipv6->proto = IPPROTO_GRE;
3090 case RTE_FLOW_ITEM_TYPE_VOID:
3093 return rte_flow_error_set(error, EINVAL,
3094 RTE_FLOW_ERROR_TYPE_ACTION,
3095 (void *)items->type,
3096 "unsupported item type");
3106 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3108 struct rte_ether_hdr *eth = NULL;
3109 struct rte_vlan_hdr *vlan = NULL;
3110 struct rte_ipv6_hdr *ipv6 = NULL;
3111 struct rte_udp_hdr *udp = NULL;
3115 eth = (struct rte_ether_hdr *)data;
3116 next_hdr = (char *)(eth + 1);
3117 proto = RTE_BE16(eth->ether_type);
3120 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3121 vlan = (struct rte_vlan_hdr *)next_hdr;
3122 proto = RTE_BE16(vlan->eth_proto);
3123 next_hdr += sizeof(struct rte_vlan_hdr);
3126 /* HW calculates IPv4 csum. no need to proceed */
3127 if (proto == RTE_ETHER_TYPE_IPV4)
3130 /* non IPv4/IPv6 header. not supported */
3131 if (proto != RTE_ETHER_TYPE_IPV6) {
3132 return rte_flow_error_set(error, ENOTSUP,
3133 RTE_FLOW_ERROR_TYPE_ACTION,
3134 NULL, "Cannot offload non IPv4/IPv6");
3137 ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3139 /* ignore non UDP */
3140 if (ipv6->proto != IPPROTO_UDP)
3143 udp = (struct rte_udp_hdr *)(ipv6 + 1);
3144 udp->dgram_cksum = 0;
3150 * Convert L2 encap action to DV specification.
3153 * Pointer to rte_eth_dev structure.
3155 * Pointer to action structure.
3156 * @param[in, out] dev_flow
3157 * Pointer to the mlx5_flow.
3158 * @param[in] transfer
3159 * Mark if the flow is E-Switch flow.
3161 * Pointer to the error structure.
3164 * 0 on success, a negative errno value otherwise and rte_errno is set.
3167 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3168 const struct rte_flow_action *action,
3169 struct mlx5_flow *dev_flow,
3171 struct rte_flow_error *error)
3173 const struct rte_flow_item *encap_data;
3174 const struct rte_flow_action_raw_encap *raw_encap_data;
3175 struct mlx5_flow_dv_encap_decap_resource res = {
3177 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3178 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3179 MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3182 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3184 (const struct rte_flow_action_raw_encap *)action->conf;
3185 res.size = raw_encap_data->size;
3186 memcpy(res.buf, raw_encap_data->data, res.size);
3188 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3190 ((const struct rte_flow_action_vxlan_encap *)
3191 action->conf)->definition;
3194 ((const struct rte_flow_action_nvgre_encap *)
3195 action->conf)->definition;
3196 if (flow_dv_convert_encap_data(encap_data, res.buf,
3200 if (flow_dv_zero_encap_udp_csum(res.buf, error))
3202 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3203 return rte_flow_error_set(error, EINVAL,
3204 RTE_FLOW_ERROR_TYPE_ACTION,
3205 NULL, "can't create L2 encap action");
3210 * Convert L2 decap action to DV specification.
3213 * Pointer to rte_eth_dev structure.
3214 * @param[in, out] dev_flow
3215 * Pointer to the mlx5_flow.
3216 * @param[in] transfer
3217 * Mark if the flow is E-Switch flow.
3219 * Pointer to the error structure.
3222 * 0 on success, a negative errno value otherwise and rte_errno is set.
3225 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3226 struct mlx5_flow *dev_flow,
3228 struct rte_flow_error *error)
3230 struct mlx5_flow_dv_encap_decap_resource res = {
3233 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3234 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3235 MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3238 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3239 return rte_flow_error_set(error, EINVAL,
3240 RTE_FLOW_ERROR_TYPE_ACTION,
3241 NULL, "can't create L2 decap action");
3246 * Convert raw decap/encap (L3 tunnel) action to DV specification.
3249 * Pointer to rte_eth_dev structure.
3251 * Pointer to action structure.
3252 * @param[in, out] dev_flow
3253 * Pointer to the mlx5_flow.
3255 * Pointer to the flow attributes.
3257 * Pointer to the error structure.
3260 * 0 on success, a negative errno value otherwise and rte_errno is set.
3263 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3264 const struct rte_flow_action *action,
3265 struct mlx5_flow *dev_flow,
3266 const struct rte_flow_attr *attr,
3267 struct rte_flow_error *error)
3269 const struct rte_flow_action_raw_encap *encap_data;
3270 struct mlx5_flow_dv_encap_decap_resource res;
3272 memset(&res, 0, sizeof(res));
3273 encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3274 res.size = encap_data->size;
3275 memcpy(res.buf, encap_data->data, res.size);
3276 res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3277 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3278 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3280 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3282 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3283 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3284 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3285 return rte_flow_error_set(error, EINVAL,
3286 RTE_FLOW_ERROR_TYPE_ACTION,
3287 NULL, "can't create encap action");
3292 * Create action push VLAN.
3295 * Pointer to rte_eth_dev structure.
3297 * Pointer to the flow attributes.
3299 * Pointer to the vlan to push to the Ethernet header.
3300 * @param[in, out] dev_flow
3301 * Pointer to the mlx5_flow.
3303 * Pointer to the error structure.
3306 * 0 on success, a negative errno value otherwise and rte_errno is set.
3309 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3310 const struct rte_flow_attr *attr,
3311 const struct rte_vlan_hdr *vlan,
3312 struct mlx5_flow *dev_flow,
3313 struct rte_flow_error *error)
3315 struct mlx5_flow_dv_push_vlan_action_resource res;
3317 memset(&res, 0, sizeof(res));
3319 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3322 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3324 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3325 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3326 return flow_dv_push_vlan_action_resource_register
3327 (dev, &res, dev_flow, error);
3331 * Validate the modify-header actions.
3333 * @param[in] action_flags
3334 * Holds the actions detected until now.
3336 * Pointer to the modify action.
3338 * Pointer to error structure.
3341 * 0 on success, a negative errno value otherwise and rte_errno is set.
3344 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3345 const struct rte_flow_action *action,
3346 struct rte_flow_error *error)
3348 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3349 return rte_flow_error_set(error, EINVAL,
3350 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3351 NULL, "action configuration not set");
3352 if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3353 return rte_flow_error_set(error, EINVAL,
3354 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3355 "can't have encap action before"
3361 * Validate the modify-header MAC address actions.
3363 * @param[in] action_flags
3364 * Holds the actions detected until now.
3366 * Pointer to the modify action.
3367 * @param[in] item_flags
3368 * Holds the items detected.
3370 * Pointer to error structure.
3373 * 0 on success, a negative errno value otherwise and rte_errno is set.
3376 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3377 const struct rte_flow_action *action,
3378 const uint64_t item_flags,
3379 struct rte_flow_error *error)
3383 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3385 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3386 return rte_flow_error_set(error, EINVAL,
3387 RTE_FLOW_ERROR_TYPE_ACTION,
3389 "no L2 item in pattern");
3395 * Validate the modify-header IPv4 address actions.
3397 * @param[in] action_flags
3398 * Holds the actions detected until now.
3400 * Pointer to the modify action.
3401 * @param[in] item_flags
3402 * Holds the items detected.
3404 * Pointer to error structure.
3407 * 0 on success, a negative errno value otherwise and rte_errno is set.
3410 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3411 const struct rte_flow_action *action,
3412 const uint64_t item_flags,
3413 struct rte_flow_error *error)
3418 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3420 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3421 MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3422 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3423 if (!(item_flags & layer))
3424 return rte_flow_error_set(error, EINVAL,
3425 RTE_FLOW_ERROR_TYPE_ACTION,
3427 "no ipv4 item in pattern");
3433 * Validate the modify-header IPv6 address actions.
3435 * @param[in] action_flags
3436 * Holds the actions detected until now.
3438 * Pointer to the modify action.
3439 * @param[in] item_flags
3440 * Holds the items detected.
3442 * Pointer to error structure.
3445 * 0 on success, a negative errno value otherwise and rte_errno is set.
3448 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3449 const struct rte_flow_action *action,
3450 const uint64_t item_flags,
3451 struct rte_flow_error *error)
3456 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3458 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3459 MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3460 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3461 if (!(item_flags & layer))
3462 return rte_flow_error_set(error, EINVAL,
3463 RTE_FLOW_ERROR_TYPE_ACTION,
3465 "no ipv6 item in pattern");
3471 * Validate the modify-header TP actions.
3473 * @param[in] action_flags
3474 * Holds the actions detected until now.
3476 * Pointer to the modify action.
3477 * @param[in] item_flags
3478 * Holds the items detected.
3480 * Pointer to error structure.
3483 * 0 on success, a negative errno value otherwise and rte_errno is set.
3486 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3487 const struct rte_flow_action *action,
3488 const uint64_t item_flags,
3489 struct rte_flow_error *error)
3494 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3496 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3497 MLX5_FLOW_LAYER_INNER_L4 :
3498 MLX5_FLOW_LAYER_OUTER_L4;
3499 if (!(item_flags & layer))
3500 return rte_flow_error_set(error, EINVAL,
3501 RTE_FLOW_ERROR_TYPE_ACTION,
3502 NULL, "no transport layer "
3509 * Validate the modify-header actions of increment/decrement
3510 * TCP Sequence-number.
3512 * @param[in] action_flags
3513 * Holds the actions detected until now.
3515 * Pointer to the modify action.
3516 * @param[in] item_flags
3517 * Holds the items detected.
3519 * Pointer to error structure.
3522 * 0 on success, a negative errno value otherwise and rte_errno is set.
3525 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3526 const struct rte_flow_action *action,
3527 const uint64_t item_flags,
3528 struct rte_flow_error *error)
3533 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3535 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3536 MLX5_FLOW_LAYER_INNER_L4_TCP :
3537 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3538 if (!(item_flags & layer))
3539 return rte_flow_error_set(error, EINVAL,
3540 RTE_FLOW_ERROR_TYPE_ACTION,
3541 NULL, "no TCP item in"
3543 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3544 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3545 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3546 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3547 return rte_flow_error_set(error, EINVAL,
3548 RTE_FLOW_ERROR_TYPE_ACTION,
3550 "cannot decrease and increase"
3551 " TCP sequence number"
3552 " at the same time");
3558 * Validate the modify-header actions of increment/decrement
3559 * TCP Acknowledgment number.
3561 * @param[in] action_flags
3562 * Holds the actions detected until now.
3564 * Pointer to the modify action.
3565 * @param[in] item_flags
3566 * Holds the items detected.
3568 * Pointer to error structure.
3571 * 0 on success, a negative errno value otherwise and rte_errno is set.
3574 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3575 const struct rte_flow_action *action,
3576 const uint64_t item_flags,
3577 struct rte_flow_error *error)
3582 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3584 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3585 MLX5_FLOW_LAYER_INNER_L4_TCP :
3586 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3587 if (!(item_flags & layer))
3588 return rte_flow_error_set(error, EINVAL,
3589 RTE_FLOW_ERROR_TYPE_ACTION,
3590 NULL, "no TCP item in"
3592 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3593 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3594 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3595 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3596 return rte_flow_error_set(error, EINVAL,
3597 RTE_FLOW_ERROR_TYPE_ACTION,
3599 "cannot decrease and increase"
3600 " TCP acknowledgment number"
3601 " at the same time");
3607 * Validate the modify-header TTL actions.
3609 * @param[in] action_flags
3610 * Holds the actions detected until now.
3612 * Pointer to the modify action.
3613 * @param[in] item_flags
3614 * Holds the items detected.
3616 * Pointer to error structure.
3619 * 0 on success, a negative errno value otherwise and rte_errno is set.
3622 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3623 const struct rte_flow_action *action,
3624 const uint64_t item_flags,
3625 struct rte_flow_error *error)
3630 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3632 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3633 MLX5_FLOW_LAYER_INNER_L3 :
3634 MLX5_FLOW_LAYER_OUTER_L3;
3635 if (!(item_flags & layer))
3636 return rte_flow_error_set(error, EINVAL,
3637 RTE_FLOW_ERROR_TYPE_ACTION,
3639 "no IP protocol in pattern");
3645 * Validate jump action.
3648 * Pointer to the jump action.
3649 * @param[in] action_flags
3650 * Holds the actions detected until now.
3651 * @param[in] attributes
3652 * Pointer to flow attributes
3653 * @param[in] external
3654 * Action belongs to flow rule created by request external to PMD.
3656 * Pointer to error structure.
3659 * 0 on success, a negative errno value otherwise and rte_errno is set.
3662 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3663 uint64_t action_flags,
3664 const struct rte_flow_attr *attributes,
3665 bool external, struct rte_flow_error *error)
3667 uint32_t target_group, table;
3670 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3671 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3672 return rte_flow_error_set(error, EINVAL,
3673 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3674 "can't have 2 fate actions in"
3676 if (action_flags & MLX5_FLOW_ACTION_METER)
3677 return rte_flow_error_set(error, ENOTSUP,
3678 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3679 "jump with meter not support");
3681 return rte_flow_error_set(error, EINVAL,
3682 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3683 NULL, "action configuration not set");
3685 ((const struct rte_flow_action_jump *)action->conf)->group;
3686 ret = mlx5_flow_group_to_table(attributes, external, target_group,
3687 true, &table, error);
3690 if (attributes->group == target_group)
3691 return rte_flow_error_set(error, EINVAL,
3692 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3693 "target group must be other than"
3694 " the current flow group");
3699 * Validate the port_id action.
3702 * Pointer to rte_eth_dev structure.
3703 * @param[in] action_flags
3704 * Bit-fields that holds the actions detected until now.
3706 * Port_id RTE action structure.
3708 * Attributes of flow that includes this action.
3710 * Pointer to error structure.
3713 * 0 on success, a negative errno value otherwise and rte_errno is set.
3716 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3717 uint64_t action_flags,
3718 const struct rte_flow_action *action,
3719 const struct rte_flow_attr *attr,
3720 struct rte_flow_error *error)
3722 const struct rte_flow_action_port_id *port_id;
3723 struct mlx5_priv *act_priv;
3724 struct mlx5_priv *dev_priv;
3727 if (!attr->transfer)
3728 return rte_flow_error_set(error, ENOTSUP,
3729 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3731 "port id action is valid in transfer"
3733 if (!action || !action->conf)
3734 return rte_flow_error_set(error, ENOTSUP,
3735 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3737 "port id action parameters must be"
3739 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3740 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3741 return rte_flow_error_set(error, EINVAL,
3742 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3743 "can have only one fate actions in"
3745 dev_priv = mlx5_dev_to_eswitch_info(dev);
3747 return rte_flow_error_set(error, rte_errno,
3748 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3750 "failed to obtain E-Switch info");
3751 port_id = action->conf;
3752 port = port_id->original ? dev->data->port_id : port_id->id;
3753 act_priv = mlx5_port_to_eswitch_info(port, false);
3755 return rte_flow_error_set
3757 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3758 "failed to obtain E-Switch port id for port");
3759 if (act_priv->domain_id != dev_priv->domain_id)
3760 return rte_flow_error_set
3762 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3763 "port does not belong to"
3764 " E-Switch being configured");
3769 * Get the maximum number of modify header actions.
3772 * Pointer to rte_eth_dev structure.
3774 * Flags bits to check if root level.
3777 * Max number of modify header actions device can support.
3779 static inline unsigned int
3780 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3784 * There's no way to directly query the max capacity from FW.
3785 * The maximal value on root table should be assumed to be supported.
3787 if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3788 return MLX5_MAX_MODIFY_NUM;
3790 return MLX5_ROOT_TBL_MODIFY_NUM;
3794 * Validate the meter action.
3797 * Pointer to rte_eth_dev structure.
3798 * @param[in] action_flags
3799 * Bit-fields that holds the actions detected until now.
3801 * Pointer to the meter action.
3803 * Attributes of flow that includes this action.
3805 * Pointer to error structure.
3808 * 0 on success, a negative errno value otherwise and rte_ernno is set.
3811 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3812 uint64_t action_flags,
3813 const struct rte_flow_action *action,
3814 const struct rte_flow_attr *attr,
3815 struct rte_flow_error *error)
3817 struct mlx5_priv *priv = dev->data->dev_private;
3818 const struct rte_flow_action_meter *am = action->conf;
3819 struct mlx5_flow_meter *fm;
3822 return rte_flow_error_set(error, EINVAL,
3823 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3824 "meter action conf is NULL");
3826 if (action_flags & MLX5_FLOW_ACTION_METER)
3827 return rte_flow_error_set(error, ENOTSUP,
3828 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3829 "meter chaining not support");
3830 if (action_flags & MLX5_FLOW_ACTION_JUMP)
3831 return rte_flow_error_set(error, ENOTSUP,
3832 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3833 "meter with jump not support");
3835 return rte_flow_error_set(error, ENOTSUP,
3836 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3838 "meter action not supported");
3839 fm = mlx5_flow_meter_find(priv, am->mtr_id);
3841 return rte_flow_error_set(error, EINVAL,
3842 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3844 if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3845 (!fm->ingress && !attr->ingress && attr->egress) ||
3846 (!fm->egress && !attr->egress && attr->ingress))))
3847 return rte_flow_error_set(error, EINVAL,
3848 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3849 "Flow attributes are either invalid "
3850 "or have a conflict with current "
3851 "meter attributes");
3856 * Validate the age action.
3858 * @param[in] action_flags
3859 * Holds the actions detected until now.
3861 * Pointer to the age action.
3863 * Pointer to the Ethernet device structure.
3865 * Pointer to error structure.
3868 * 0 on success, a negative errno value otherwise and rte_errno is set.
3871 flow_dv_validate_action_age(uint64_t action_flags,
3872 const struct rte_flow_action *action,
3873 struct rte_eth_dev *dev,
3874 struct rte_flow_error *error)
3876 struct mlx5_priv *priv = dev->data->dev_private;
3877 const struct rte_flow_action_age *age = action->conf;
3879 if (!priv->config.devx || priv->counter_fallback)
3880 return rte_flow_error_set(error, ENOTSUP,
3881 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3883 "age action not supported");
3884 if (!(action->conf))
3885 return rte_flow_error_set(error, EINVAL,
3886 RTE_FLOW_ERROR_TYPE_ACTION, action,
3887 "configuration cannot be null");
3888 if (age->timeout >= UINT16_MAX / 2 / 10)
3889 return rte_flow_error_set(error, ENOTSUP,
3890 RTE_FLOW_ERROR_TYPE_ACTION, action,
3891 "Max age time: 3275 seconds");
3892 if (action_flags & MLX5_FLOW_ACTION_AGE)
3893 return rte_flow_error_set(error, EINVAL,
3894 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3895 "Duplicate age ctions set");
3900 * Validate the modify-header IPv4 DSCP actions.
3902 * @param[in] action_flags
3903 * Holds the actions detected until now.
3905 * Pointer to the modify action.
3906 * @param[in] item_flags
3907 * Holds the items detected.
3909 * Pointer to error structure.
3912 * 0 on success, a negative errno value otherwise and rte_errno is set.
3915 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3916 const struct rte_flow_action *action,
3917 const uint64_t item_flags,
3918 struct rte_flow_error *error)
3922 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3924 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3925 return rte_flow_error_set(error, EINVAL,
3926 RTE_FLOW_ERROR_TYPE_ACTION,
3928 "no ipv4 item in pattern");
3934 * Validate the modify-header IPv6 DSCP actions.
3936 * @param[in] action_flags
3937 * Holds the actions detected until now.
3939 * Pointer to the modify action.
3940 * @param[in] item_flags
3941 * Holds the items detected.
3943 * Pointer to error structure.
3946 * 0 on success, a negative errno value otherwise and rte_errno is set.
3949 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3950 const struct rte_flow_action *action,
3951 const uint64_t item_flags,
3952 struct rte_flow_error *error)
3956 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3958 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3959 return rte_flow_error_set(error, EINVAL,
3960 RTE_FLOW_ERROR_TYPE_ACTION,
3962 "no ipv6 item in pattern");
3968 * Find existing modify-header resource or create and register a new one.
3970 * @param dev[in, out]
3971 * Pointer to rte_eth_dev structure.
3972 * @param[in, out] resource
3973 * Pointer to modify-header resource.
3974 * @parm[in, out] dev_flow
3975 * Pointer to the dev_flow.
3977 * pointer to error structure.
3980 * 0 on success otherwise -errno and errno is set.
3983 flow_dv_modify_hdr_resource_register
3984 (struct rte_eth_dev *dev,
3985 struct mlx5_flow_dv_modify_hdr_resource *resource,
3986 struct mlx5_flow *dev_flow,
3987 struct rte_flow_error *error)
3989 struct mlx5_priv *priv = dev->data->dev_private;
3990 struct mlx5_dev_ctx_shared *sh = priv->sh;
3991 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3992 struct mlx5dv_dr_domain *ns;
3993 uint32_t actions_len;
3996 resource->flags = dev_flow->dv.group ? 0 :
3997 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
3998 if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4000 return rte_flow_error_set(error, EOVERFLOW,
4001 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4002 "too many modify header items");
4003 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4004 ns = sh->fdb_domain;
4005 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4009 /* Lookup a matching resource from cache. */
4010 actions_len = resource->actions_num * sizeof(resource->actions[0]);
4011 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
4012 if (resource->ft_type == cache_resource->ft_type &&
4013 resource->actions_num == cache_resource->actions_num &&
4014 resource->flags == cache_resource->flags &&
4015 !memcmp((const void *)resource->actions,
4016 (const void *)cache_resource->actions,
4018 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4019 (void *)cache_resource,
4020 rte_atomic32_read(&cache_resource->refcnt));
4021 rte_atomic32_inc(&cache_resource->refcnt);
4022 dev_flow->handle->dvh.modify_hdr = cache_resource;
4026 /* Register new modify-header resource. */
4027 cache_resource = rte_calloc(__func__, 1,
4028 sizeof(*cache_resource) + actions_len, 0);
4029 if (!cache_resource)
4030 return rte_flow_error_set(error, ENOMEM,
4031 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4032 "cannot allocate resource memory");
4033 *cache_resource = *resource;
4034 rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4035 ret = mlx5_flow_os_create_flow_action_modify_header
4036 (sh->ctx, ns, cache_resource,
4037 actions_len, &cache_resource->action);
4039 rte_free(cache_resource);
4040 return rte_flow_error_set(error, ENOMEM,
4041 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4042 NULL, "cannot create action");
4044 rte_atomic32_init(&cache_resource->refcnt);
4045 rte_atomic32_inc(&cache_resource->refcnt);
4046 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4047 dev_flow->handle->dvh.modify_hdr = cache_resource;
4048 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4049 (void *)cache_resource,
4050 rte_atomic32_read(&cache_resource->refcnt));
4055 * Get DV flow counter by index.
4058 * Pointer to the Ethernet device structure.
4060 * mlx5 flow counter index in the container.
4062 * mlx5 flow counter pool in the container,
4065 * Pointer to the counter, NULL otherwise.
4067 static struct mlx5_flow_counter *
4068 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4070 struct mlx5_flow_counter_pool **ppool)
4072 struct mlx5_priv *priv = dev->data->dev_private;
4073 struct mlx5_pools_container *cont;
4074 struct mlx5_flow_counter_pool *pool;
4075 uint32_t batch = 0, age = 0;
4078 age = MLX_CNT_IS_AGE(idx);
4079 idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4080 if (idx >= MLX5_CNT_BATCH_OFFSET) {
4081 idx -= MLX5_CNT_BATCH_OFFSET;
4084 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4085 MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4086 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4090 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4094 * Check the devx counter belongs to the pool.
4097 * Pointer to the counter pool.
4099 * The counter devx ID.
4102 * True if counter belongs to the pool, false otherwise.
4105 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4107 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4108 MLX5_COUNTERS_PER_POOL;
4110 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4116 * Get a pool by devx counter ID.
4119 * Pointer to the counter container.
4121 * The counter devx ID.
4124 * The counter pool pointer if exists, NULL otherwise,
4126 static struct mlx5_flow_counter_pool *
4127 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4131 /* Check last used pool. */
4132 if (cont->last_pool_idx != POOL_IDX_INVALID &&
4133 flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4134 return cont->pools[cont->last_pool_idx];
4135 /* ID out of range means no suitable pool in the container. */
4136 if (id > cont->max_id || id < cont->min_id)
4139 * Find the pool from the end of the container, since mostly counter
4140 * ID is sequence increasing, and the last pool should be the needed
4143 i = rte_atomic16_read(&cont->n_valid);
4145 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4147 if (flow_dv_is_counter_in_pool(pool, id))
4154 * Allocate a new memory for the counter values wrapped by all the needed
4158 * Pointer to the Ethernet device structure.
4160 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4163 * The new memory management pointer on success, otherwise NULL and rte_errno
4166 static struct mlx5_counter_stats_mem_mng *
4167 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4169 struct mlx5_priv *priv = dev->data->dev_private;
4170 struct mlx5_dev_ctx_shared *sh = priv->sh;
4171 struct mlx5_devx_mkey_attr mkey_attr;
4172 struct mlx5_counter_stats_mem_mng *mem_mng;
4173 volatile struct flow_counter_stats *raw_data;
4174 int size = (sizeof(struct flow_counter_stats) *
4175 MLX5_COUNTERS_PER_POOL +
4176 sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4177 sizeof(struct mlx5_counter_stats_mem_mng);
4178 uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
4185 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4186 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4187 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4188 IBV_ACCESS_LOCAL_WRITE);
4189 if (!mem_mng->umem) {
4194 mkey_attr.addr = (uintptr_t)mem;
4195 mkey_attr.size = size;
4196 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4197 mkey_attr.pd = sh->pdn;
4198 mkey_attr.log_entity_size = 0;
4199 mkey_attr.pg_access = 0;
4200 mkey_attr.klm_array = NULL;
4201 mkey_attr.klm_num = 0;
4202 if (priv->config.hca_attr.relaxed_ordering_write &&
4203 priv->config.hca_attr.relaxed_ordering_read &&
4204 !haswell_broadwell_cpu)
4205 mkey_attr.relaxed_ordering = 1;
4206 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4208 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4213 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4214 raw_data = (volatile struct flow_counter_stats *)mem;
4215 for (i = 0; i < raws_n; ++i) {
4216 mem_mng->raws[i].mem_mng = mem_mng;
4217 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4219 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4224 * Resize a counter container.
4227 * Pointer to the Ethernet device structure.
4229 * Whether the pool is for counter that was allocated by batch command.
4231 * Whether the pool is for Aging counter.
4234 * 0 on success, otherwise negative errno value and rte_errno is set.
4237 flow_dv_container_resize(struct rte_eth_dev *dev,
4238 uint32_t batch, uint32_t age)
4240 struct mlx5_priv *priv = dev->data->dev_private;
4241 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4243 struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4244 void *old_pools = cont->pools;
4245 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4246 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4247 void *pools = rte_calloc(__func__, 1, mem_size, 0);
4254 memcpy(pools, old_pools, cont->n *
4255 sizeof(struct mlx5_flow_counter_pool *));
4257 * Fallback mode query the counter directly, no background query
4258 * resources are needed.
4260 if (!priv->counter_fallback) {
4263 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4264 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4269 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4270 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4272 MLX5_CNT_CONTAINER_RESIZE +
4275 rte_spinlock_lock(&cont->resize_sl);
4277 cont->mem_mng = mem_mng;
4278 cont->pools = pools;
4279 rte_spinlock_unlock(&cont->resize_sl);
4281 rte_free(old_pools);
4286 * Query a devx flow counter.
4289 * Pointer to the Ethernet device structure.
4291 * Index to the flow counter.
4293 * The statistics value of packets.
4295 * The statistics value of bytes.
4298 * 0 on success, otherwise a negative errno value and rte_errno is set.
4301 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4304 struct mlx5_priv *priv = dev->data->dev_private;
4305 struct mlx5_flow_counter_pool *pool = NULL;
4306 struct mlx5_flow_counter *cnt;
4307 struct mlx5_flow_counter_ext *cnt_ext = NULL;
4310 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4312 if (counter < MLX5_CNT_BATCH_OFFSET) {
4313 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4314 if (priv->counter_fallback)
4315 return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4316 0, pkts, bytes, 0, NULL, NULL, 0);
4319 rte_spinlock_lock(&pool->sl);
4321 * The single counters allocation may allocate smaller ID than the
4322 * current allocated in parallel to the host reading.
4323 * In this case the new counter values must be reported as 0.
4325 if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4329 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4330 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4331 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4333 rte_spinlock_unlock(&pool->sl);
4338 * Create and initialize a new counter pool.
4341 * Pointer to the Ethernet device structure.
4343 * The devX counter handle.
4345 * Whether the pool is for counter that was allocated by batch command.
4347 * Whether the pool is for counter that was allocated for aging.
4348 * @param[in/out] cont_cur
4349 * Pointer to the container pointer, it will be update in pool resize.
4352 * The pool container pointer on success, NULL otherwise and rte_errno is set.
4354 static struct mlx5_flow_counter_pool *
4355 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4356 uint32_t batch, uint32_t age)
4358 struct mlx5_priv *priv = dev->data->dev_private;
4359 struct mlx5_flow_counter_pool *pool;
4360 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4362 int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4363 uint32_t size = sizeof(*pool);
4365 if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4367 size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4368 size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4369 size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4370 pool = rte_calloc(__func__, 1, size, 0);
4375 pool->min_dcs = dcs;
4376 if (!priv->counter_fallback)
4377 pool->raw = cont->mem_mng->raws + n_valid %
4378 MLX5_CNT_CONTAINER_RESIZE;
4379 pool->raw_hw = NULL;
4381 pool->type |= (batch ? 0 : CNT_POOL_TYPE_EXT);
4382 pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE);
4383 pool->query_gen = 0;
4384 rte_spinlock_init(&pool->sl);
4385 TAILQ_INIT(&pool->counters[0]);
4386 TAILQ_INIT(&pool->counters[1]);
4387 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4388 pool->index = n_valid;
4389 cont->pools[n_valid] = pool;
4391 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4393 if (base < cont->min_id)
4394 cont->min_id = base;
4395 if (base > cont->max_id)
4396 cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4397 cont->last_pool_idx = pool->index;
4399 /* Pool initialization must be updated before host thread access. */
4401 rte_atomic16_add(&cont->n_valid, 1);
4406 * Update the minimum dcs-id for aged or no-aged counter pool.
4409 * Pointer to the Ethernet device structure.
4411 * Current counter pool.
4413 * Whether the pool is for counter that was allocated by batch command.
4415 * Whether the counter is for aging.
4418 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4419 struct mlx5_flow_counter_pool *pool,
4420 uint32_t batch, uint32_t age)
4422 struct mlx5_priv *priv = dev->data->dev_private;
4423 struct mlx5_flow_counter_pool *other;
4424 struct mlx5_pools_container *cont;
4426 cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4427 other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4430 if (pool->min_dcs->id < other->min_dcs->id) {
4431 rte_atomic64_set(&other->a64_dcs,
4432 rte_atomic64_read(&pool->a64_dcs));
4434 rte_atomic64_set(&pool->a64_dcs,
4435 rte_atomic64_read(&other->a64_dcs));
4439 * Prepare a new counter and/or a new counter pool.
4442 * Pointer to the Ethernet device structure.
4443 * @param[out] cnt_free
4444 * Where to put the pointer of a new counter.
4446 * Whether the pool is for counter that was allocated by batch command.
4448 * Whether the pool is for counter that was allocated for aging.
4451 * The counter pool pointer and @p cnt_free is set on success,
4452 * NULL otherwise and rte_errno is set.
4454 static struct mlx5_flow_counter_pool *
4455 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4456 struct mlx5_flow_counter **cnt_free,
4457 uint32_t batch, uint32_t age)
4459 struct mlx5_priv *priv = dev->data->dev_private;
4460 struct mlx5_pools_container *cont;
4461 struct mlx5_flow_counter_pool *pool;
4462 struct mlx5_counters tmp_tq;
4463 struct mlx5_devx_obj *dcs = NULL;
4464 struct mlx5_flow_counter *cnt;
4467 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4469 /* bulk_bitmap must be 0 for single counter allocation. */
4470 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4473 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4475 pool = flow_dv_pool_create(dev, dcs, batch, age);
4477 mlx5_devx_cmd_destroy(dcs);
4480 } else if (dcs->id < pool->min_dcs->id) {
4481 rte_atomic64_set(&pool->a64_dcs,
4482 (int64_t)(uintptr_t)dcs);
4484 flow_dv_counter_update_min_dcs(dev,
4486 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4487 cnt = MLX5_POOL_GET_CNT(pool, i);
4489 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4493 /* bulk_bitmap is in 128 counters units. */
4494 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4495 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4497 rte_errno = ENODATA;
4500 pool = flow_dv_pool_create(dev, dcs, batch, age);
4502 mlx5_devx_cmd_destroy(dcs);
4505 TAILQ_INIT(&tmp_tq);
4506 for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4507 cnt = MLX5_POOL_GET_CNT(pool, i);
4509 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4511 rte_spinlock_lock(&cont->csl);
4512 TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4513 rte_spinlock_unlock(&cont->csl);
4514 *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4515 (*cnt_free)->pool = pool;
4520 * Search for existed shared counter.
4523 * Pointer to the Ethernet device structure.
4525 * The shared counter ID to search.
4527 * mlx5 flow counter pool in the container,
4530 * NULL if not existed, otherwise pointer to the shared extend counter.
4532 static struct mlx5_flow_counter_ext *
4533 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4534 struct mlx5_flow_counter_pool **ppool)
4536 struct mlx5_priv *priv = dev->data->dev_private;
4537 union mlx5_l3t_data data;
4540 if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4542 cnt_idx = data.dword;
4544 * Shared counters don't have age info. The counter extend is after
4545 * the counter datat structure.
4547 return (struct mlx5_flow_counter_ext *)
4548 ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4552 * Allocate a flow counter.
4555 * Pointer to the Ethernet device structure.
4557 * Indicate if this counter is shared with other flows.
4559 * Counter identifier.
4561 * Counter flow group.
4563 * Whether the counter was allocated for aging.
4566 * Index to flow counter on success, 0 otherwise and rte_errno is set.
4569 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4570 uint16_t group, uint32_t age)
4572 struct mlx5_priv *priv = dev->data->dev_private;
4573 struct mlx5_flow_counter_pool *pool = NULL;
4574 struct mlx5_flow_counter *cnt_free = NULL;
4575 struct mlx5_flow_counter_ext *cnt_ext = NULL;
4577 * Currently group 0 flow counter cannot be assigned to a flow if it is
4578 * not the first one in the batch counter allocation, so it is better
4579 * to allocate counters one by one for these flows in a separate
4581 * A counter can be shared between different groups so need to take
4582 * shared counters from the single container.
4584 uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4585 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4589 if (!priv->config.devx) {
4590 rte_errno = ENOTSUP;
4594 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4596 if (cnt_ext->ref_cnt + 1 == 0) {
4601 cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4602 (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4607 /* Get free counters from container. */
4608 rte_spinlock_lock(&cont->csl);
4609 cnt_free = TAILQ_FIRST(&cont->counters);
4611 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4612 rte_spinlock_unlock(&cont->csl);
4613 if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4616 pool = cnt_free->pool;
4618 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4619 /* Create a DV counter action only in the first time usage. */
4620 if (!cnt_free->action) {
4622 struct mlx5_devx_obj *dcs;
4626 offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4627 dcs = pool->min_dcs;
4632 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4639 cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4640 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4641 cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4642 cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4643 /* Update the counter reset values. */
4644 if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4648 cnt_ext->shared = shared;
4649 cnt_ext->ref_cnt = 1;
4652 union mlx5_l3t_data data;
4654 data.dword = cnt_idx;
4655 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4659 if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4660 /* Start the asynchronous batch query by the host thread. */
4661 mlx5_set_query_alarm(priv->sh);
4665 cnt_free->pool = pool;
4666 rte_spinlock_lock(&cont->csl);
4667 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4668 rte_spinlock_unlock(&cont->csl);
4674 * Get age param from counter index.
4677 * Pointer to the Ethernet device structure.
4678 * @param[in] counter
4679 * Index to the counter handler.
4682 * The aging parameter specified for the counter index.
4684 static struct mlx5_age_param*
4685 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4688 struct mlx5_flow_counter *cnt;
4689 struct mlx5_flow_counter_pool *pool = NULL;
4691 flow_dv_counter_get_by_idx(dev, counter, &pool);
4692 counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4693 cnt = MLX5_POOL_GET_CNT(pool, counter);
4694 return MLX5_CNT_TO_AGE(cnt);
4698 * Remove a flow counter from aged counter list.
4701 * Pointer to the Ethernet device structure.
4702 * @param[in] counter
4703 * Index to the counter handler.
4705 * Pointer to the counter handler.
4708 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4709 uint32_t counter, struct mlx5_flow_counter *cnt)
4711 struct mlx5_age_info *age_info;
4712 struct mlx5_age_param *age_param;
4713 struct mlx5_priv *priv = dev->data->dev_private;
4715 age_info = GET_PORT_AGE_INFO(priv);
4716 age_param = flow_dv_counter_idx_get_age(dev, counter);
4717 if (rte_atomic16_cmpset((volatile uint16_t *)
4719 AGE_CANDIDATE, AGE_FREE)
4722 * We need the lock even it is age timeout,
4723 * since counter may still in process.
4725 rte_spinlock_lock(&age_info->aged_sl);
4726 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4727 rte_spinlock_unlock(&age_info->aged_sl);
4729 rte_atomic16_set(&age_param->state, AGE_FREE);
4732 * Release a flow counter.
4735 * Pointer to the Ethernet device structure.
4736 * @param[in] counter
4737 * Index to the counter handler.
4740 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4742 struct mlx5_priv *priv = dev->data->dev_private;
4743 struct mlx5_flow_counter_pool *pool = NULL;
4744 struct mlx5_flow_counter *cnt;
4745 struct mlx5_flow_counter_ext *cnt_ext = NULL;
4749 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4751 if (counter < MLX5_CNT_BATCH_OFFSET) {
4752 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4754 if (--cnt_ext->ref_cnt)
4756 if (cnt_ext->shared)
4757 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4761 if (IS_AGE_POOL(pool))
4762 flow_dv_counter_remove_from_age(dev, counter, cnt);
4765 * Put the counter back to list to be updated in none fallback mode.
4766 * Currently, we are using two list alternately, while one is in query,
4767 * add the freed counter to the other list based on the pool query_gen
4768 * value. After query finishes, add counter the list to the global
4769 * container counter list. The list changes while query starts. In
4770 * this case, lock will not be needed as query callback and release
4771 * function both operate with the different list.
4774 if (!priv->counter_fallback)
4775 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4777 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4778 (priv->sh, 0, 0))->counters),
4783 * Verify the @p attributes will be correctly understood by the NIC and store
4784 * them in the @p flow if everything is correct.
4787 * Pointer to dev struct.
4788 * @param[in] attributes
4789 * Pointer to flow attributes
4790 * @param[in] external
4791 * This flow rule is created by request external to PMD.
4793 * Pointer to error structure.
4796 * - 0 on success and non root table.
4797 * - 1 on success and root table.
4798 * - a negative errno value otherwise and rte_errno is set.
4801 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4802 const struct rte_flow_attr *attributes,
4803 bool external __rte_unused,
4804 struct rte_flow_error *error)
4806 struct mlx5_priv *priv = dev->data->dev_private;
4807 uint32_t priority_max = priv->config.flow_prio - 1;
4810 #ifndef HAVE_MLX5DV_DR
4811 if (attributes->group)
4812 return rte_flow_error_set(error, ENOTSUP,
4813 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4815 "groups are not supported");
4819 ret = mlx5_flow_group_to_table(attributes, external,
4820 attributes->group, !!priv->fdb_def_rule,
4825 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4827 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4828 attributes->priority >= priority_max)
4829 return rte_flow_error_set(error, ENOTSUP,
4830 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4832 "priority out of range");
4833 if (attributes->transfer) {
4834 if (!priv->config.dv_esw_en)
4835 return rte_flow_error_set
4837 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4838 "E-Switch dr is not supported");
4839 if (!(priv->representor || priv->master))
4840 return rte_flow_error_set
4841 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4842 NULL, "E-Switch configuration can only be"
4843 " done by a master or a representor device");
4844 if (attributes->egress)
4845 return rte_flow_error_set
4847 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4848 "egress is not supported");
4850 if (!(attributes->egress ^ attributes->ingress))
4851 return rte_flow_error_set(error, ENOTSUP,
4852 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4853 "must specify exactly one of "
4854 "ingress or egress");
4859 * Internal validation function. For validating both actions and items.
4862 * Pointer to the rte_eth_dev structure.
4864 * Pointer to the flow attributes.
4866 * Pointer to the list of items.
4867 * @param[in] actions
4868 * Pointer to the list of actions.
4869 * @param[in] external
4870 * This flow rule is created by request external to PMD.
4871 * @param[in] hairpin
4872 * Number of hairpin TX actions, 0 means classic flow.
4874 * Pointer to the error structure.
4877 * 0 on success, a negative errno value otherwise and rte_errno is set.
4880 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4881 const struct rte_flow_item items[],
4882 const struct rte_flow_action actions[],
4883 bool external, int hairpin, struct rte_flow_error *error)
4886 uint64_t action_flags = 0;
4887 uint64_t item_flags = 0;
4888 uint64_t last_item = 0;
4889 uint8_t next_protocol = 0xff;
4890 uint16_t ether_type = 0;
4892 uint8_t item_ipv6_proto = 0;
4893 const struct rte_flow_item *gre_item = NULL;
4894 const struct rte_flow_action_raw_decap *decap;
4895 const struct rte_flow_action_raw_encap *encap;
4896 const struct rte_flow_action_rss *rss;
4897 const struct rte_flow_item_tcp nic_tcp_mask = {
4900 .src_port = RTE_BE16(UINT16_MAX),
4901 .dst_port = RTE_BE16(UINT16_MAX),
4904 const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4906 .src_addr = RTE_BE32(0xffffffff),
4907 .dst_addr = RTE_BE32(0xffffffff),
4908 .type_of_service = 0xff,
4909 .next_proto_id = 0xff,
4910 .time_to_live = 0xff,
4913 const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4916 "\xff\xff\xff\xff\xff\xff\xff\xff"
4917 "\xff\xff\xff\xff\xff\xff\xff\xff",
4919 "\xff\xff\xff\xff\xff\xff\xff\xff"
4920 "\xff\xff\xff\xff\xff\xff\xff\xff",
4921 .vtc_flow = RTE_BE32(0xffffffff),
4926 const struct rte_flow_item_ecpri nic_ecpri_mask = {
4930 RTE_BE32(((const struct rte_ecpri_common_hdr) {
4934 .dummy[0] = 0xffffffff,
4937 struct mlx5_priv *priv = dev->data->dev_private;
4938 struct mlx5_dev_config *dev_conf = &priv->config;
4939 uint16_t queue_index = 0xFFFF;
4940 const struct rte_flow_item_vlan *vlan_m = NULL;
4941 int16_t rw_act_num = 0;
4946 ret = flow_dv_validate_attributes(dev, attr, external, error);
4949 is_root = (uint64_t)ret;
4950 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4951 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4952 int type = items->type;
4954 if (!mlx5_flow_os_item_supported(type))
4955 return rte_flow_error_set(error, ENOTSUP,
4956 RTE_FLOW_ERROR_TYPE_ITEM,
4957 NULL, "item not supported");
4959 case RTE_FLOW_ITEM_TYPE_VOID:
4961 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4962 ret = flow_dv_validate_item_port_id
4963 (dev, items, attr, item_flags, error);
4966 last_item = MLX5_FLOW_ITEM_PORT_ID;
4968 case RTE_FLOW_ITEM_TYPE_ETH:
4969 ret = mlx5_flow_validate_item_eth(items, item_flags,
4973 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4974 MLX5_FLOW_LAYER_OUTER_L2;
4975 if (items->mask != NULL && items->spec != NULL) {
4977 ((const struct rte_flow_item_eth *)
4980 ((const struct rte_flow_item_eth *)
4982 ether_type = rte_be_to_cpu_16(ether_type);
4987 case RTE_FLOW_ITEM_TYPE_VLAN:
4988 ret = flow_dv_validate_item_vlan(items, item_flags,
4992 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
4993 MLX5_FLOW_LAYER_OUTER_VLAN;
4994 if (items->mask != NULL && items->spec != NULL) {
4996 ((const struct rte_flow_item_vlan *)
4997 items->spec)->inner_type;
4999 ((const struct rte_flow_item_vlan *)
5000 items->mask)->inner_type;
5001 ether_type = rte_be_to_cpu_16(ether_type);
5005 /* Store outer VLAN mask for of_push_vlan action. */
5007 vlan_m = items->mask;
5009 case RTE_FLOW_ITEM_TYPE_IPV4:
5010 mlx5_flow_tunnel_ip_check(items, next_protocol,
5011 &item_flags, &tunnel);
5012 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
5019 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5020 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5021 if (items->mask != NULL &&
5022 ((const struct rte_flow_item_ipv4 *)
5023 items->mask)->hdr.next_proto_id) {
5025 ((const struct rte_flow_item_ipv4 *)
5026 (items->spec))->hdr.next_proto_id;
5028 ((const struct rte_flow_item_ipv4 *)
5029 (items->mask))->hdr.next_proto_id;
5031 /* Reset for inner layer. */
5032 next_protocol = 0xff;
5035 case RTE_FLOW_ITEM_TYPE_IPV6:
5036 mlx5_flow_tunnel_ip_check(items, next_protocol,
5037 &item_flags, &tunnel);
5038 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5045 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5046 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5047 if (items->mask != NULL &&
5048 ((const struct rte_flow_item_ipv6 *)
5049 items->mask)->hdr.proto) {
5051 ((const struct rte_flow_item_ipv6 *)
5052 items->spec)->hdr.proto;
5054 ((const struct rte_flow_item_ipv6 *)
5055 items->spec)->hdr.proto;
5057 ((const struct rte_flow_item_ipv6 *)
5058 items->mask)->hdr.proto;
5060 /* Reset for inner layer. */
5061 next_protocol = 0xff;
5064 case RTE_FLOW_ITEM_TYPE_TCP:
5065 ret = mlx5_flow_validate_item_tcp
5072 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5073 MLX5_FLOW_LAYER_OUTER_L4_TCP;
5075 case RTE_FLOW_ITEM_TYPE_UDP:
5076 ret = mlx5_flow_validate_item_udp(items, item_flags,
5081 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5082 MLX5_FLOW_LAYER_OUTER_L4_UDP;
5084 case RTE_FLOW_ITEM_TYPE_GRE:
5085 ret = mlx5_flow_validate_item_gre(items, item_flags,
5086 next_protocol, error);
5090 last_item = MLX5_FLOW_LAYER_GRE;
5092 case RTE_FLOW_ITEM_TYPE_NVGRE:
5093 ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5098 last_item = MLX5_FLOW_LAYER_NVGRE;
5100 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5101 ret = mlx5_flow_validate_item_gre_key
5102 (items, item_flags, gre_item, error);
5105 last_item = MLX5_FLOW_LAYER_GRE_KEY;
5107 case RTE_FLOW_ITEM_TYPE_VXLAN:
5108 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5112 last_item = MLX5_FLOW_LAYER_VXLAN;
5114 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5115 ret = mlx5_flow_validate_item_vxlan_gpe(items,
5120 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5122 case RTE_FLOW_ITEM_TYPE_GENEVE:
5123 ret = mlx5_flow_validate_item_geneve(items,
5128 last_item = MLX5_FLOW_LAYER_GENEVE;
5130 case RTE_FLOW_ITEM_TYPE_MPLS:
5131 ret = mlx5_flow_validate_item_mpls(dev, items,
5136 last_item = MLX5_FLOW_LAYER_MPLS;
5139 case RTE_FLOW_ITEM_TYPE_MARK:
5140 ret = flow_dv_validate_item_mark(dev, items, attr,
5144 last_item = MLX5_FLOW_ITEM_MARK;
5146 case RTE_FLOW_ITEM_TYPE_META:
5147 ret = flow_dv_validate_item_meta(dev, items, attr,
5151 last_item = MLX5_FLOW_ITEM_METADATA;
5153 case RTE_FLOW_ITEM_TYPE_ICMP:
5154 ret = mlx5_flow_validate_item_icmp(items, item_flags,
5159 last_item = MLX5_FLOW_LAYER_ICMP;
5161 case RTE_FLOW_ITEM_TYPE_ICMP6:
5162 ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5167 item_ipv6_proto = IPPROTO_ICMPV6;
5168 last_item = MLX5_FLOW_LAYER_ICMP6;
5170 case RTE_FLOW_ITEM_TYPE_TAG:
5171 ret = flow_dv_validate_item_tag(dev, items,
5175 last_item = MLX5_FLOW_ITEM_TAG;
5177 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5178 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5180 case RTE_FLOW_ITEM_TYPE_GTP:
5181 ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5185 last_item = MLX5_FLOW_LAYER_GTP;
5187 case RTE_FLOW_ITEM_TYPE_ECPRI:
5188 /* Capacity will be checked in the translate stage. */
5189 ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5196 last_item = MLX5_FLOW_LAYER_ECPRI;
5199 return rte_flow_error_set(error, ENOTSUP,
5200 RTE_FLOW_ERROR_TYPE_ITEM,
5201 NULL, "item not supported");
5203 item_flags |= last_item;
5205 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5206 int type = actions->type;
5208 if (!mlx5_flow_os_action_supported(type))
5209 return rte_flow_error_set(error, ENOTSUP,
5210 RTE_FLOW_ERROR_TYPE_ACTION,
5212 "action not supported");
5213 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5214 return rte_flow_error_set(error, ENOTSUP,
5215 RTE_FLOW_ERROR_TYPE_ACTION,
5216 actions, "too many actions");
5218 case RTE_FLOW_ACTION_TYPE_VOID:
5220 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5221 ret = flow_dv_validate_action_port_id(dev,
5228 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5231 case RTE_FLOW_ACTION_TYPE_FLAG:
5232 ret = flow_dv_validate_action_flag(dev, action_flags,
5236 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5237 /* Count all modify-header actions as one. */
5238 if (!(action_flags &
5239 MLX5_FLOW_MODIFY_HDR_ACTIONS))
5241 action_flags |= MLX5_FLOW_ACTION_FLAG |
5242 MLX5_FLOW_ACTION_MARK_EXT;
5244 action_flags |= MLX5_FLOW_ACTION_FLAG;
5247 rw_act_num += MLX5_ACT_NUM_SET_MARK;
5249 case RTE_FLOW_ACTION_TYPE_MARK:
5250 ret = flow_dv_validate_action_mark(dev, actions,
5255 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5256 /* Count all modify-header actions as one. */
5257 if (!(action_flags &
5258 MLX5_FLOW_MODIFY_HDR_ACTIONS))
5260 action_flags |= MLX5_FLOW_ACTION_MARK |
5261 MLX5_FLOW_ACTION_MARK_EXT;
5263 action_flags |= MLX5_FLOW_ACTION_MARK;
5266 rw_act_num += MLX5_ACT_NUM_SET_MARK;
5268 case RTE_FLOW_ACTION_TYPE_SET_META:
5269 ret = flow_dv_validate_action_set_meta(dev, actions,
5274 /* Count all modify-header actions as one action. */
5275 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5277 action_flags |= MLX5_FLOW_ACTION_SET_META;
5278 rw_act_num += MLX5_ACT_NUM_SET_META;
5280 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5281 ret = flow_dv_validate_action_set_tag(dev, actions,
5286 /* Count all modify-header actions as one action. */
5287 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5289 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5290 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5292 case RTE_FLOW_ACTION_TYPE_DROP:
5293 ret = mlx5_flow_validate_action_drop(action_flags,
5297 action_flags |= MLX5_FLOW_ACTION_DROP;
5300 case RTE_FLOW_ACTION_TYPE_QUEUE:
5301 ret = mlx5_flow_validate_action_queue(actions,
5306 queue_index = ((const struct rte_flow_action_queue *)
5307 (actions->conf))->index;
5308 action_flags |= MLX5_FLOW_ACTION_QUEUE;
5311 case RTE_FLOW_ACTION_TYPE_RSS:
5312 rss = actions->conf;
5313 ret = mlx5_flow_validate_action_rss(actions,
5319 if (rss != NULL && rss->queue_num)
5320 queue_index = rss->queue[0];
5321 action_flags |= MLX5_FLOW_ACTION_RSS;
5324 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5326 mlx5_flow_validate_action_default_miss(action_flags,
5330 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5333 case RTE_FLOW_ACTION_TYPE_COUNT:
5334 ret = flow_dv_validate_action_count(dev, error);
5337 action_flags |= MLX5_FLOW_ACTION_COUNT;
5340 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5341 if (flow_dv_validate_action_pop_vlan(dev,
5347 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5350 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5351 ret = flow_dv_validate_action_push_vlan(dev,
5358 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5361 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5362 ret = flow_dv_validate_action_set_vlan_pcp
5363 (action_flags, actions, error);
5366 /* Count PCP with push_vlan command. */
5367 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5369 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5370 ret = flow_dv_validate_action_set_vlan_vid
5371 (item_flags, action_flags,
5375 /* Count VID with push_vlan command. */
5376 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5377 rw_act_num += MLX5_ACT_NUM_MDF_VID;
5379 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5380 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5381 ret = flow_dv_validate_action_l2_encap(dev,
5387 action_flags |= MLX5_FLOW_ACTION_ENCAP;
5390 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5391 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5392 ret = flow_dv_validate_action_decap(dev, action_flags,
5396 action_flags |= MLX5_FLOW_ACTION_DECAP;
5399 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5400 ret = flow_dv_validate_action_raw_encap_decap
5401 (dev, NULL, actions->conf, attr, &action_flags,
5406 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5407 decap = actions->conf;
5408 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5410 if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5414 encap = actions->conf;
5416 ret = flow_dv_validate_action_raw_encap_decap
5418 decap ? decap : &empty_decap, encap,
5419 attr, &action_flags, &actions_n,
5424 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5425 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5426 ret = flow_dv_validate_action_modify_mac(action_flags,
5432 /* Count all modify-header actions as one action. */
5433 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5435 action_flags |= actions->type ==
5436 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5437 MLX5_FLOW_ACTION_SET_MAC_SRC :
5438 MLX5_FLOW_ACTION_SET_MAC_DST;
5440 * Even if the source and destination MAC addresses have
5441 * overlap in the header with 4B alignment, the convert
5442 * function will handle them separately and 4 SW actions
5443 * will be created. And 2 actions will be added each
5444 * time no matter how many bytes of address will be set.
5446 rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5448 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5449 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5450 ret = flow_dv_validate_action_modify_ipv4(action_flags,
5456 /* Count all modify-header actions as one action. */
5457 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5459 action_flags |= actions->type ==
5460 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5461 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5462 MLX5_FLOW_ACTION_SET_IPV4_DST;
5463 rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5465 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5466 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5467 ret = flow_dv_validate_action_modify_ipv6(action_flags,
5473 if (item_ipv6_proto == IPPROTO_ICMPV6)
5474 return rte_flow_error_set(error, ENOTSUP,
5475 RTE_FLOW_ERROR_TYPE_ACTION,
5477 "Can't change header "
5478 "with ICMPv6 proto");
5479 /* Count all modify-header actions as one action. */
5480 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5482 action_flags |= actions->type ==
5483 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5484 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5485 MLX5_FLOW_ACTION_SET_IPV6_DST;
5486 rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5488 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5489 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5490 ret = flow_dv_validate_action_modify_tp(action_flags,
5496 /* Count all modify-header actions as one action. */
5497 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5499 action_flags |= actions->type ==
5500 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5501 MLX5_FLOW_ACTION_SET_TP_SRC :
5502 MLX5_FLOW_ACTION_SET_TP_DST;
5503 rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5505 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5506 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5507 ret = flow_dv_validate_action_modify_ttl(action_flags,
5513 /* Count all modify-header actions as one action. */
5514 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5516 action_flags |= actions->type ==
5517 RTE_FLOW_ACTION_TYPE_SET_TTL ?
5518 MLX5_FLOW_ACTION_SET_TTL :
5519 MLX5_FLOW_ACTION_DEC_TTL;
5520 rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5522 case RTE_FLOW_ACTION_TYPE_JUMP:
5523 ret = flow_dv_validate_action_jump(actions,
5530 action_flags |= MLX5_FLOW_ACTION_JUMP;
5532 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5533 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5534 ret = flow_dv_validate_action_modify_tcp_seq
5541 /* Count all modify-header actions as one action. */
5542 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5544 action_flags |= actions->type ==
5545 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5546 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5547 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5548 rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5550 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5551 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5552 ret = flow_dv_validate_action_modify_tcp_ack
5559 /* Count all modify-header actions as one action. */
5560 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5562 action_flags |= actions->type ==
5563 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5564 MLX5_FLOW_ACTION_INC_TCP_ACK :
5565 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5566 rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5568 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5570 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5571 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5572 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5574 case RTE_FLOW_ACTION_TYPE_METER:
5575 ret = mlx5_flow_validate_action_meter(dev,
5581 action_flags |= MLX5_FLOW_ACTION_METER;
5583 /* Meter action will add one more TAG action. */
5584 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5586 case RTE_FLOW_ACTION_TYPE_AGE:
5587 ret = flow_dv_validate_action_age(action_flags,
5592 action_flags |= MLX5_FLOW_ACTION_AGE;
5595 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5596 ret = flow_dv_validate_action_modify_ipv4_dscp
5603 /* Count all modify-header actions as one action. */
5604 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5606 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5607 rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5609 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5610 ret = flow_dv_validate_action_modify_ipv6_dscp
5617 /* Count all modify-header actions as one action. */
5618 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5620 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5621 rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5624 return rte_flow_error_set(error, ENOTSUP,
5625 RTE_FLOW_ERROR_TYPE_ACTION,
5627 "action not supported");
5631 * Validate the drop action mutual exclusion with other actions.
5632 * Drop action is mutually-exclusive with any other action, except for
5635 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5636 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5637 return rte_flow_error_set(error, EINVAL,
5638 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5639 "Drop action is mutually-exclusive "
5640 "with any other action, except for "
5642 /* Eswitch has few restrictions on using items and actions */
5643 if (attr->transfer) {
5644 if (!mlx5_flow_ext_mreg_supported(dev) &&
5645 action_flags & MLX5_FLOW_ACTION_FLAG)
5646 return rte_flow_error_set(error, ENOTSUP,
5647 RTE_FLOW_ERROR_TYPE_ACTION,
5649 "unsupported action FLAG");
5650 if (!mlx5_flow_ext_mreg_supported(dev) &&
5651 action_flags & MLX5_FLOW_ACTION_MARK)
5652 return rte_flow_error_set(error, ENOTSUP,
5653 RTE_FLOW_ERROR_TYPE_ACTION,
5655 "unsupported action MARK");
5656 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5657 return rte_flow_error_set(error, ENOTSUP,
5658 RTE_FLOW_ERROR_TYPE_ACTION,
5660 "unsupported action QUEUE");
5661 if (action_flags & MLX5_FLOW_ACTION_RSS)
5662 return rte_flow_error_set(error, ENOTSUP,
5663 RTE_FLOW_ERROR_TYPE_ACTION,
5665 "unsupported action RSS");
5666 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5667 return rte_flow_error_set(error, EINVAL,
5668 RTE_FLOW_ERROR_TYPE_ACTION,
5670 "no fate action is found");
5672 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5673 return rte_flow_error_set(error, EINVAL,
5674 RTE_FLOW_ERROR_TYPE_ACTION,
5676 "no fate action is found");
5678 /* Continue validation for Xcap actions.*/
5679 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5680 mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5681 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5682 MLX5_FLOW_XCAP_ACTIONS)
5683 return rte_flow_error_set(error, ENOTSUP,
5684 RTE_FLOW_ERROR_TYPE_ACTION,
5685 NULL, "encap and decap "
5686 "combination aren't supported");
5687 if (!attr->transfer && attr->ingress && (action_flags &
5688 MLX5_FLOW_ACTION_ENCAP))
5689 return rte_flow_error_set(error, ENOTSUP,
5690 RTE_FLOW_ERROR_TYPE_ACTION,
5691 NULL, "encap is not supported"
5692 " for ingress traffic");
5694 /* Hairpin flow will add one more TAG action. */
5696 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5697 /* extra metadata enabled: one more TAG action will be add. */
5698 if (dev_conf->dv_flow_en &&
5699 dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5700 mlx5_flow_ext_mreg_supported(dev))
5701 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5702 if ((uint32_t)rw_act_num >
5703 flow_dv_modify_hdr_action_max(dev, is_root)) {
5704 return rte_flow_error_set(error, ENOTSUP,
5705 RTE_FLOW_ERROR_TYPE_ACTION,
5706 NULL, "too many header modify"
5707 " actions to support");
5713 * Internal preparation function. Allocates the DV flow size,
5714 * this size is constant.
5717 * Pointer to the rte_eth_dev structure.
5719 * Pointer to the flow attributes.
5721 * Pointer to the list of items.
5722 * @param[in] actions
5723 * Pointer to the list of actions.
5725 * Pointer to the error structure.
5728 * Pointer to mlx5_flow object on success,
5729 * otherwise NULL and rte_errno is set.
5731 static struct mlx5_flow *
5732 flow_dv_prepare(struct rte_eth_dev *dev,
5733 const struct rte_flow_attr *attr __rte_unused,
5734 const struct rte_flow_item items[] __rte_unused,
5735 const struct rte_flow_action actions[] __rte_unused,
5736 struct rte_flow_error *error)
5738 uint32_t handle_idx = 0;
5739 struct mlx5_flow *dev_flow;
5740 struct mlx5_flow_handle *dev_handle;
5741 struct mlx5_priv *priv = dev->data->dev_private;
5743 /* In case of corrupting the memory. */
5744 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5745 rte_flow_error_set(error, ENOSPC,
5746 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5747 "not free temporary device flow");
5750 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5753 rte_flow_error_set(error, ENOMEM,
5754 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5755 "not enough memory to create flow handle");
5758 /* No multi-thread supporting. */
5759 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5760 dev_flow->handle = dev_handle;
5761 dev_flow->handle_idx = handle_idx;
5762 dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
5764 * The matching value needs to be cleared to 0 before using. In the
5765 * past, it will be automatically cleared when using rte_*alloc
5766 * API. The time consumption will be almost the same as before.
5768 memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5769 dev_flow->ingress = attr->ingress;
5770 dev_flow->dv.transfer = attr->transfer;
5774 #ifdef RTE_LIBRTE_MLX5_DEBUG
5776 * Sanity check for match mask and value. Similar to check_valid_spec() in
5777 * kernel driver. If unmasked bit is present in value, it returns failure.
5780 * pointer to match mask buffer.
5781 * @param match_value
5782 * pointer to match value buffer.
5785 * 0 if valid, -EINVAL otherwise.
5788 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5790 uint8_t *m = match_mask;
5791 uint8_t *v = match_value;
5794 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5797 "match_value differs from match_criteria"
5798 " %p[%u] != %p[%u]",
5799 match_value, i, match_mask, i);
5808 * Add match of ip_version.
5812 * @param[in] headers_v
5813 * Values header pointer.
5814 * @param[in] headers_m
5815 * Masks header pointer.
5816 * @param[in] ip_version
5817 * The IP version to set.
5820 flow_dv_set_match_ip_version(uint32_t group,
5826 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5828 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5830 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5831 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5832 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5836 * Add Ethernet item to matcher and to the value.
5838 * @param[in, out] matcher
5840 * @param[in, out] key
5841 * Flow matcher value.
5843 * Flow pattern to translate.
5845 * Item is inner pattern.
5848 flow_dv_translate_item_eth(void *matcher, void *key,
5849 const struct rte_flow_item *item, int inner,
5852 const struct rte_flow_item_eth *eth_m = item->mask;
5853 const struct rte_flow_item_eth *eth_v = item->spec;
5854 const struct rte_flow_item_eth nic_mask = {
5855 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5856 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5857 .type = RTE_BE16(0xffff),
5869 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5871 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5873 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5875 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5877 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5878 ð_m->dst, sizeof(eth_m->dst));
5879 /* The value must be in the range of the mask. */
5880 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5881 for (i = 0; i < sizeof(eth_m->dst); ++i)
5882 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5883 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5884 ð_m->src, sizeof(eth_m->src));
5885 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5886 /* The value must be in the range of the mask. */
5887 for (i = 0; i < sizeof(eth_m->dst); ++i)
5888 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5890 /* When ethertype is present set mask for tagged VLAN. */
5891 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5892 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5893 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5894 eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5895 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5897 /* Return here to avoid setting match on ethertype. */
5902 * HW supports match on one Ethertype, the Ethertype following the last
5903 * VLAN tag of the packet (see PRM).
5904 * Set match on ethertype only if ETH header is not followed by VLAN.
5905 * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5906 * ethertype, and use ip_version field instead.
5907 * eCPRI over Ether layer will use type value 0xAEFE.
5909 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5910 eth_m->type == 0xFFFF) {
5911 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5912 } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5913 eth_m->type == 0xFFFF) {
5914 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5916 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5917 rte_be_to_cpu_16(eth_m->type));
5918 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5920 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5925 * Add VLAN item to matcher and to the value.
5927 * @param[in, out] dev_flow
5929 * @param[in, out] matcher
5931 * @param[in, out] key
5932 * Flow matcher value.
5934 * Flow pattern to translate.
5936 * Item is inner pattern.
5939 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5940 void *matcher, void *key,
5941 const struct rte_flow_item *item,
5942 int inner, uint32_t group)
5944 const struct rte_flow_item_vlan *vlan_m = item->mask;
5945 const struct rte_flow_item_vlan *vlan_v = item->spec;
5952 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5954 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5956 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5958 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5960 * This is workaround, masks are not supported,
5961 * and pre-validated.
5964 dev_flow->handle->vf_vlan.tag =
5965 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5968 * When VLAN item exists in flow, mark packet as tagged,
5969 * even if TCI is not specified.
5971 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5972 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5976 vlan_m = &rte_flow_item_vlan_mask;
5977 tci_m = rte_be_to_cpu_16(vlan_m->tci);
5978 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5979 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5980 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5981 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5982 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5983 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
5984 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
5986 * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5987 * ethertype, and use ip_version field instead.
5989 if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5990 vlan_m->inner_type == 0xFFFF) {
5991 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5992 } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5993 vlan_m->inner_type == 0xFFFF) {
5994 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5996 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5997 rte_be_to_cpu_16(vlan_m->inner_type));
5998 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
5999 rte_be_to_cpu_16(vlan_m->inner_type &
6000 vlan_v->inner_type));
6005 * Add IPV4 item to matcher and to the value.
6007 * @param[in, out] matcher
6009 * @param[in, out] key
6010 * Flow matcher value.
6012 * Flow pattern to translate.
6013 * @param[in] item_flags
6014 * Bit-fields that holds the items detected until now.
6016 * Item is inner pattern.
6018 * The group to insert the rule.
6021 flow_dv_translate_item_ipv4(void *matcher, void *key,
6022 const struct rte_flow_item *item,
6023 const uint64_t item_flags,
6024 int inner, uint32_t group)
6026 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6027 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6028 const struct rte_flow_item_ipv4 nic_mask = {
6030 .src_addr = RTE_BE32(0xffffffff),
6031 .dst_addr = RTE_BE32(0xffffffff),
6032 .type_of_service = 0xff,
6033 .next_proto_id = 0xff,
6034 .time_to_live = 0xff,
6044 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6046 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6048 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6050 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6052 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6054 * On outer header (which must contains L2), or inner header with L2,
6055 * set cvlan_tag mask bit to mark this packet as untagged.
6056 * This should be done even if item->spec is empty.
6058 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6059 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6064 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6065 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6066 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6067 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6068 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6069 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6070 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6071 src_ipv4_src_ipv6.ipv4_layout.ipv4);
6072 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6073 src_ipv4_src_ipv6.ipv4_layout.ipv4);
6074 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6075 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6076 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6077 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6078 ipv4_m->hdr.type_of_service);
6079 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6080 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6081 ipv4_m->hdr.type_of_service >> 2);
6082 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6083 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6084 ipv4_m->hdr.next_proto_id);
6085 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6086 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6087 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6088 ipv4_m->hdr.time_to_live);
6089 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6090 ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6094 * Add IPV6 item to matcher and to the value.
6096 * @param[in, out] matcher
6098 * @param[in, out] key
6099 * Flow matcher value.
6101 * Flow pattern to translate.
6102 * @param[in] item_flags
6103 * Bit-fields that holds the items detected until now.
6105 * Item is inner pattern.
6107 * The group to insert the rule.
6110 flow_dv_translate_item_ipv6(void *matcher, void *key,
6111 const struct rte_flow_item *item,
6112 const uint64_t item_flags,
6113 int inner, uint32_t group)
6115 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6116 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6117 const struct rte_flow_item_ipv6 nic_mask = {
6120 "\xff\xff\xff\xff\xff\xff\xff\xff"
6121 "\xff\xff\xff\xff\xff\xff\xff\xff",
6123 "\xff\xff\xff\xff\xff\xff\xff\xff"
6124 "\xff\xff\xff\xff\xff\xff\xff\xff",
6125 .vtc_flow = RTE_BE32(0xffffffff),
6132 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6133 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6142 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6144 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6146 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6148 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6150 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6152 * On outer header (which must contains L2), or inner header with L2,
6153 * set cvlan_tag mask bit to mark this packet as untagged.
6154 * This should be done even if item->spec is empty.
6156 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6157 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6162 size = sizeof(ipv6_m->hdr.dst_addr);
6163 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6164 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6165 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6166 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6167 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6168 for (i = 0; i < size; ++i)
6169 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6170 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6171 src_ipv4_src_ipv6.ipv6_layout.ipv6);
6172 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6173 src_ipv4_src_ipv6.ipv6_layout.ipv6);
6174 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6175 for (i = 0; i < size; ++i)
6176 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6178 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6179 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6180 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6181 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6182 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6183 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6186 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6188 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6191 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6193 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6197 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6199 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6200 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6202 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6203 ipv6_m->hdr.hop_limits);
6204 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6205 ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6209 * Add TCP item to matcher and to the value.
6211 * @param[in, out] matcher
6213 * @param[in, out] key
6214 * Flow matcher value.
6216 * Flow pattern to translate.
6218 * Item is inner pattern.
6221 flow_dv_translate_item_tcp(void *matcher, void *key,
6222 const struct rte_flow_item *item,
6225 const struct rte_flow_item_tcp *tcp_m = item->mask;
6226 const struct rte_flow_item_tcp *tcp_v = item->spec;
6231 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6233 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6235 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6237 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6239 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6240 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6244 tcp_m = &rte_flow_item_tcp_mask;
6245 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6246 rte_be_to_cpu_16(tcp_m->hdr.src_port));
6247 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6248 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6249 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6250 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6251 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6252 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6253 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6254 tcp_m->hdr.tcp_flags);
6255 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6256 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6260 * Add UDP item to matcher and to the value.
6262 * @param[in, out] matcher
6264 * @param[in, out] key
6265 * Flow matcher value.
6267 * Flow pattern to translate.
6269 * Item is inner pattern.
6272 flow_dv_translate_item_udp(void *matcher, void *key,
6273 const struct rte_flow_item *item,
6276 const struct rte_flow_item_udp *udp_m = item->mask;
6277 const struct rte_flow_item_udp *udp_v = item->spec;
6282 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6284 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6286 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6288 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6290 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6291 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6295 udp_m = &rte_flow_item_udp_mask;
6296 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6297 rte_be_to_cpu_16(udp_m->hdr.src_port));
6298 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6299 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6300 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6301 rte_be_to_cpu_16(udp_m->hdr.dst_port));
6302 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6303 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6307 * Add GRE optional Key item to matcher and to the value.
6309 * @param[in, out] matcher
6311 * @param[in, out] key
6312 * Flow matcher value.
6314 * Flow pattern to translate.
6316 * Item is inner pattern.
6319 flow_dv_translate_item_gre_key(void *matcher, void *key,
6320 const struct rte_flow_item *item)
6322 const rte_be32_t *key_m = item->mask;
6323 const rte_be32_t *key_v = item->spec;
6324 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6325 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6326 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6328 /* GRE K bit must be on and should already be validated */
6329 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6330 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6334 key_m = &gre_key_default_mask;
6335 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6336 rte_be_to_cpu_32(*key_m) >> 8);
6337 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6338 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6339 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6340 rte_be_to_cpu_32(*key_m) & 0xFF);
6341 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6342 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6346 * Add GRE item to matcher and to the value.
6348 * @param[in, out] matcher
6350 * @param[in, out] key
6351 * Flow matcher value.
6353 * Flow pattern to translate.
6355 * Item is inner pattern.
6358 flow_dv_translate_item_gre(void *matcher, void *key,
6359 const struct rte_flow_item *item,
6362 const struct rte_flow_item_gre *gre_m = item->mask;
6363 const struct rte_flow_item_gre *gre_v = item->spec;
6366 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6367 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6374 uint16_t s_present:1;
6375 uint16_t k_present:1;
6376 uint16_t rsvd_bit1:1;
6377 uint16_t c_present:1;
6381 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6384 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6386 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6388 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6390 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6392 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6393 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6397 gre_m = &rte_flow_item_gre_mask;
6398 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6399 rte_be_to_cpu_16(gre_m->protocol));
6400 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6401 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6402 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6403 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6404 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6405 gre_crks_rsvd0_ver_m.c_present);
6406 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6407 gre_crks_rsvd0_ver_v.c_present &
6408 gre_crks_rsvd0_ver_m.c_present);
6409 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6410 gre_crks_rsvd0_ver_m.k_present);
6411 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6412 gre_crks_rsvd0_ver_v.k_present &
6413 gre_crks_rsvd0_ver_m.k_present);
6414 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6415 gre_crks_rsvd0_ver_m.s_present);
6416 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6417 gre_crks_rsvd0_ver_v.s_present &
6418 gre_crks_rsvd0_ver_m.s_present);
6422 * Add NVGRE item to matcher and to the value.
6424 * @param[in, out] matcher
6426 * @param[in, out] key
6427 * Flow matcher value.
6429 * Flow pattern to translate.
6431 * Item is inner pattern.
6434 flow_dv_translate_item_nvgre(void *matcher, void *key,
6435 const struct rte_flow_item *item,
6438 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6439 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6440 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6441 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6442 const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6443 const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6449 /* For NVGRE, GRE header fields must be set with defined values. */
6450 const struct rte_flow_item_gre gre_spec = {
6451 .c_rsvd0_ver = RTE_BE16(0x2000),
6452 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6454 const struct rte_flow_item_gre gre_mask = {
6455 .c_rsvd0_ver = RTE_BE16(0xB000),
6456 .protocol = RTE_BE16(UINT16_MAX),
6458 const struct rte_flow_item gre_item = {
6463 flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6467 nvgre_m = &rte_flow_item_nvgre_mask;
6468 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6469 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6470 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6471 memcpy(gre_key_m, tni_flow_id_m, size);
6472 for (i = 0; i < size; ++i)
6473 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6477 * Add VXLAN item to matcher and to the value.
6479 * @param[in, out] matcher
6481 * @param[in, out] key
6482 * Flow matcher value.
6484 * Flow pattern to translate.
6486 * Item is inner pattern.
6489 flow_dv_translate_item_vxlan(void *matcher, void *key,
6490 const struct rte_flow_item *item,
6493 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6494 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6497 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6498 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6506 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6508 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6510 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6512 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6514 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6515 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6516 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6517 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6518 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6523 vxlan_m = &rte_flow_item_vxlan_mask;
6524 size = sizeof(vxlan_m->vni);
6525 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6526 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6527 memcpy(vni_m, vxlan_m->vni, size);
6528 for (i = 0; i < size; ++i)
6529 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6533 * Add VXLAN-GPE item to matcher and to the value.
6535 * @param[in, out] matcher
6537 * @param[in, out] key
6538 * Flow matcher value.
6540 * Flow pattern to translate.
6542 * Item is inner pattern.
6546 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6547 const struct rte_flow_item *item, int inner)
6549 const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6550 const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6554 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6556 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6562 uint8_t flags_m = 0xff;
6563 uint8_t flags_v = 0xc;
6566 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6568 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6570 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6572 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6574 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6575 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6576 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6577 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6578 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6583 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6584 size = sizeof(vxlan_m->vni);
6585 vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6586 vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6587 memcpy(vni_m, vxlan_m->vni, size);
6588 for (i = 0; i < size; ++i)
6589 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6590 if (vxlan_m->flags) {
6591 flags_m = vxlan_m->flags;
6592 flags_v = vxlan_v->flags;
6594 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6595 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6596 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6598 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6603 * Add Geneve item to matcher and to the value.
6605 * @param[in, out] matcher
6607 * @param[in, out] key
6608 * Flow matcher value.
6610 * Flow pattern to translate.
6612 * Item is inner pattern.
6616 flow_dv_translate_item_geneve(void *matcher, void *key,
6617 const struct rte_flow_item *item, int inner)
6619 const struct rte_flow_item_geneve *geneve_m = item->mask;
6620 const struct rte_flow_item_geneve *geneve_v = item->spec;
6623 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6624 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6633 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6635 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6637 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6639 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6641 dport = MLX5_UDP_PORT_GENEVE;
6642 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6643 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6644 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6649 geneve_m = &rte_flow_item_geneve_mask;
6650 size = sizeof(geneve_m->vni);
6651 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6652 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6653 memcpy(vni_m, geneve_m->vni, size);
6654 for (i = 0; i < size; ++i)
6655 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6656 MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6657 rte_be_to_cpu_16(geneve_m->protocol));
6658 MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6659 rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6660 gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6661 gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6662 MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6663 MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6664 MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6665 MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6666 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6667 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6668 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6669 MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6670 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6674 * Add MPLS item to matcher and to the value.
6676 * @param[in, out] matcher
6678 * @param[in, out] key
6679 * Flow matcher value.
6681 * Flow pattern to translate.
6682 * @param[in] prev_layer
6683 * The protocol layer indicated in previous item.
6685 * Item is inner pattern.
6688 flow_dv_translate_item_mpls(void *matcher, void *key,
6689 const struct rte_flow_item *item,
6690 uint64_t prev_layer,
6693 const uint32_t *in_mpls_m = item->mask;
6694 const uint32_t *in_mpls_v = item->spec;
6695 uint32_t *out_mpls_m = 0;
6696 uint32_t *out_mpls_v = 0;
6697 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6698 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6699 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6701 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6702 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6703 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6705 switch (prev_layer) {
6706 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6707 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6708 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6709 MLX5_UDP_PORT_MPLS);
6711 case MLX5_FLOW_LAYER_GRE:
6712 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6713 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6714 RTE_ETHER_TYPE_MPLS);
6717 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6718 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6725 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6726 switch (prev_layer) {
6727 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6729 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6730 outer_first_mpls_over_udp);
6732 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6733 outer_first_mpls_over_udp);
6735 case MLX5_FLOW_LAYER_GRE:
6737 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6738 outer_first_mpls_over_gre);
6740 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6741 outer_first_mpls_over_gre);
6744 /* Inner MPLS not over GRE is not supported. */
6747 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6751 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6757 if (out_mpls_m && out_mpls_v) {
6758 *out_mpls_m = *in_mpls_m;
6759 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6764 * Add metadata register item to matcher
6766 * @param[in, out] matcher
6768 * @param[in, out] key
6769 * Flow matcher value.
6770 * @param[in] reg_type
6771 * Type of device metadata register
6778 flow_dv_match_meta_reg(void *matcher, void *key,
6779 enum modify_reg reg_type,
6780 uint32_t data, uint32_t mask)
6783 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6785 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6791 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6792 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6795 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6796 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6800 * The metadata register C0 field might be divided into
6801 * source vport index and META item value, we should set
6802 * this field according to specified mask, not as whole one.
6804 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6806 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6807 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6810 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6813 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6814 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6817 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6818 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6821 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6822 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6825 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6826 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6829 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6830 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6833 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6834 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6837 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6838 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6847 * Add MARK item to matcher
6850 * The device to configure through.
6851 * @param[in, out] matcher
6853 * @param[in, out] key
6854 * Flow matcher value.
6856 * Flow pattern to translate.
6859 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6860 void *matcher, void *key,
6861 const struct rte_flow_item *item)
6863 struct mlx5_priv *priv = dev->data->dev_private;
6864 const struct rte_flow_item_mark *mark;
6868 mark = item->mask ? (const void *)item->mask :
6869 &rte_flow_item_mark_mask;
6870 mask = mark->id & priv->sh->dv_mark_mask;
6871 mark = (const void *)item->spec;
6873 value = mark->id & priv->sh->dv_mark_mask & mask;
6875 enum modify_reg reg;
6877 /* Get the metadata register index for the mark. */
6878 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6879 MLX5_ASSERT(reg > 0);
6880 if (reg == REG_C_0) {
6881 struct mlx5_priv *priv = dev->data->dev_private;
6882 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6883 uint32_t shl_c0 = rte_bsf32(msk_c0);
6889 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6894 * Add META item to matcher
6897 * The devich to configure through.
6898 * @param[in, out] matcher
6900 * @param[in, out] key
6901 * Flow matcher value.
6903 * Attributes of flow that includes this item.
6905 * Flow pattern to translate.
6908 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6909 void *matcher, void *key,
6910 const struct rte_flow_attr *attr,
6911 const struct rte_flow_item *item)
6913 const struct rte_flow_item_meta *meta_m;
6914 const struct rte_flow_item_meta *meta_v;
6916 meta_m = (const void *)item->mask;
6918 meta_m = &rte_flow_item_meta_mask;
6919 meta_v = (const void *)item->spec;
6922 uint32_t value = meta_v->data;
6923 uint32_t mask = meta_m->data;
6925 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6929 * In datapath code there is no endianness
6930 * coversions for perfromance reasons, all
6931 * pattern conversions are done in rte_flow.
6933 value = rte_cpu_to_be_32(value);
6934 mask = rte_cpu_to_be_32(mask);
6935 if (reg == REG_C_0) {
6936 struct mlx5_priv *priv = dev->data->dev_private;
6937 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6938 uint32_t shl_c0 = rte_bsf32(msk_c0);
6939 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6940 uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6947 MLX5_ASSERT(msk_c0);
6948 MLX5_ASSERT(!(~msk_c0 & mask));
6950 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6955 * Add vport metadata Reg C0 item to matcher
6957 * @param[in, out] matcher
6959 * @param[in, out] key
6960 * Flow matcher value.
6962 * Flow pattern to translate.
6965 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6966 uint32_t value, uint32_t mask)
6968 flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6972 * Add tag item to matcher
6975 * The devich to configure through.
6976 * @param[in, out] matcher
6978 * @param[in, out] key
6979 * Flow matcher value.
6981 * Flow pattern to translate.
6984 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
6985 void *matcher, void *key,
6986 const struct rte_flow_item *item)
6988 const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
6989 const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
6990 uint32_t mask, value;
6993 value = tag_v->data;
6994 mask = tag_m ? tag_m->data : UINT32_MAX;
6995 if (tag_v->id == REG_C_0) {
6996 struct mlx5_priv *priv = dev->data->dev_private;
6997 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6998 uint32_t shl_c0 = rte_bsf32(msk_c0);
7004 flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7008 * Add TAG item to matcher
7011 * The devich to configure through.
7012 * @param[in, out] matcher
7014 * @param[in, out] key
7015 * Flow matcher value.
7017 * Flow pattern to translate.
7020 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7021 void *matcher, void *key,
7022 const struct rte_flow_item *item)
7024 const struct rte_flow_item_tag *tag_v = item->spec;
7025 const struct rte_flow_item_tag *tag_m = item->mask;
7026 enum modify_reg reg;
7029 tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7030 /* Get the metadata register index for the tag. */
7031 reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7032 MLX5_ASSERT(reg > 0);
7033 flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7037 * Add source vport match to the specified matcher.
7039 * @param[in, out] matcher
7041 * @param[in, out] key
7042 * Flow matcher value.
7044 * Source vport value to match
7049 flow_dv_translate_item_source_vport(void *matcher, void *key,
7050 int16_t port, uint16_t mask)
7052 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7053 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7055 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7056 MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7060 * Translate port-id item to eswitch match on port-id.
7063 * The devich to configure through.
7064 * @param[in, out] matcher
7066 * @param[in, out] key
7067 * Flow matcher value.
7069 * Flow pattern to translate.
7072 * 0 on success, a negative errno value otherwise.
7075 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7076 void *key, const struct rte_flow_item *item)
7078 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7079 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7080 struct mlx5_priv *priv;
7083 mask = pid_m ? pid_m->id : 0xffff;
7084 id = pid_v ? pid_v->id : dev->data->port_id;
7085 priv = mlx5_port_to_eswitch_info(id, item == NULL);
7088 /* Translate to vport field or to metadata, depending on mode. */
7089 if (priv->vport_meta_mask)
7090 flow_dv_translate_item_meta_vport(matcher, key,
7091 priv->vport_meta_tag,
7092 priv->vport_meta_mask);
7094 flow_dv_translate_item_source_vport(matcher, key,
7095 priv->vport_id, mask);
7100 * Add ICMP6 item to matcher and to the value.
7102 * @param[in, out] matcher
7104 * @param[in, out] key
7105 * Flow matcher value.
7107 * Flow pattern to translate.
7109 * Item is inner pattern.
7112 flow_dv_translate_item_icmp6(void *matcher, void *key,
7113 const struct rte_flow_item *item,
7116 const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7117 const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7120 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7122 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7124 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7126 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7128 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7130 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7132 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7133 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7137 icmp6_m = &rte_flow_item_icmp6_mask;
7139 * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7140 * If only the protocol is specified, no need to match the frag.
7142 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7143 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7144 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7145 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7146 icmp6_v->type & icmp6_m->type);
7147 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7148 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7149 icmp6_v->code & icmp6_m->code);
7153 * Add ICMP item to matcher and to the value.
7155 * @param[in, out] matcher
7157 * @param[in, out] key
7158 * Flow matcher value.
7160 * Flow pattern to translate.
7162 * Item is inner pattern.
7165 flow_dv_translate_item_icmp(void *matcher, void *key,
7166 const struct rte_flow_item *item,
7169 const struct rte_flow_item_icmp *icmp_m = item->mask;
7170 const struct rte_flow_item_icmp *icmp_v = item->spec;
7173 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7175 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7177 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7179 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7181 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7183 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7185 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7186 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7190 icmp_m = &rte_flow_item_icmp_mask;
7192 * Force flow only to match the non-fragmented IPv4 ICMP packets.
7193 * If only the protocol is specified, no need to match the frag.
7195 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7196 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7197 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7198 icmp_m->hdr.icmp_type);
7199 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7200 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7201 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7202 icmp_m->hdr.icmp_code);
7203 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7204 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7208 * Add GTP item to matcher and to the value.
7210 * @param[in, out] matcher
7212 * @param[in, out] key
7213 * Flow matcher value.
7215 * Flow pattern to translate.
7217 * Item is inner pattern.
7220 flow_dv_translate_item_gtp(void *matcher, void *key,
7221 const struct rte_flow_item *item, int inner)
7223 const struct rte_flow_item_gtp *gtp_m = item->mask;
7224 const struct rte_flow_item_gtp *gtp_v = item->spec;
7227 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7229 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7230 uint16_t dport = RTE_GTPU_UDP_PORT;
7233 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7235 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7237 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7239 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7241 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7242 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7243 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7248 gtp_m = &rte_flow_item_gtp_mask;
7249 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7250 gtp_m->v_pt_rsv_flags);
7251 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7252 gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7253 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7254 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7255 gtp_v->msg_type & gtp_m->msg_type);
7256 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7257 rte_be_to_cpu_32(gtp_m->teid));
7258 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7259 rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7262 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7264 #define HEADER_IS_ZERO(match_criteria, headers) \
7265 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
7266 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7269 * Calculate flow matcher enable bitmap.
7271 * @param match_criteria
7272 * Pointer to flow matcher criteria.
7275 * Bitmap of enabled fields.
7278 flow_dv_matcher_enable(uint32_t *match_criteria)
7280 uint8_t match_criteria_enable;
7282 match_criteria_enable =
7283 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7284 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7285 match_criteria_enable |=
7286 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7287 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7288 match_criteria_enable |=
7289 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7290 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7291 match_criteria_enable |=
7292 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7293 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7294 match_criteria_enable |=
7295 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7296 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7297 return match_criteria_enable;
7304 * @param[in, out] dev
7305 * Pointer to rte_eth_dev structure.
7306 * @param[in] table_id
7309 * Direction of the table.
7310 * @param[in] transfer
7311 * E-Switch or NIC flow.
7313 * pointer to error structure.
7316 * Returns tables resource based on the index, NULL in case of failed.
7318 static struct mlx5_flow_tbl_resource *
7319 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7320 uint32_t table_id, uint8_t egress,
7322 struct rte_flow_error *error)
7324 struct mlx5_priv *priv = dev->data->dev_private;
7325 struct mlx5_dev_ctx_shared *sh = priv->sh;
7326 struct mlx5_flow_tbl_resource *tbl;
7327 union mlx5_flow_tbl_key table_key = {
7329 .table_id = table_id,
7331 .domain = !!transfer,
7332 .direction = !!egress,
7335 struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7337 struct mlx5_flow_tbl_data_entry *tbl_data;
7343 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7345 tbl = &tbl_data->tbl;
7346 rte_atomic32_inc(&tbl->refcnt);
7349 tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7351 rte_flow_error_set(error, ENOMEM,
7352 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7354 "cannot allocate flow table data entry");
7357 tbl_data->idx = idx;
7358 tbl = &tbl_data->tbl;
7359 pos = &tbl_data->entry;
7361 domain = sh->fdb_domain;
7363 domain = sh->tx_domain;
7365 domain = sh->rx_domain;
7366 ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7368 rte_flow_error_set(error, ENOMEM,
7369 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7370 NULL, "cannot create flow table object");
7371 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7375 * No multi-threads now, but still better to initialize the reference
7376 * count before insert it into the hash list.
7378 rte_atomic32_init(&tbl->refcnt);
7379 /* Jump action reference count is initialized here. */
7380 rte_atomic32_init(&tbl_data->jump.refcnt);
7381 pos->key = table_key.v64;
7382 ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7384 rte_flow_error_set(error, -ret,
7385 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7386 "cannot insert flow table data entry");
7387 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7388 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7390 rte_atomic32_inc(&tbl->refcnt);
7395 * Release a flow table.
7398 * Pointer to rte_eth_dev structure.
7400 * Table resource to be released.
7403 * Returns 0 if table was released, else return 1;
7406 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7407 struct mlx5_flow_tbl_resource *tbl)
7409 struct mlx5_priv *priv = dev->data->dev_private;
7410 struct mlx5_dev_ctx_shared *sh = priv->sh;
7411 struct mlx5_flow_tbl_data_entry *tbl_data =
7412 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7416 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7417 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7419 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7421 /* remove the entry from the hash list and free memory. */
7422 mlx5_hlist_remove(sh->flow_tbls, pos);
7423 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7431 * Register the flow matcher.
7433 * @param[in, out] dev
7434 * Pointer to rte_eth_dev structure.
7435 * @param[in, out] matcher
7436 * Pointer to flow matcher.
7437 * @param[in, out] key
7438 * Pointer to flow table key.
7439 * @parm[in, out] dev_flow
7440 * Pointer to the dev_flow.
7442 * pointer to error structure.
7445 * 0 on success otherwise -errno and errno is set.
7448 flow_dv_matcher_register(struct rte_eth_dev *dev,
7449 struct mlx5_flow_dv_matcher *matcher,
7450 union mlx5_flow_tbl_key *key,
7451 struct mlx5_flow *dev_flow,
7452 struct rte_flow_error *error)
7454 struct mlx5_priv *priv = dev->data->dev_private;
7455 struct mlx5_dev_ctx_shared *sh = priv->sh;
7456 struct mlx5_flow_dv_matcher *cache_matcher;
7457 struct mlx5dv_flow_matcher_attr dv_attr = {
7458 .type = IBV_FLOW_ATTR_NORMAL,
7459 .match_mask = (void *)&matcher->mask,
7461 struct mlx5_flow_tbl_resource *tbl;
7462 struct mlx5_flow_tbl_data_entry *tbl_data;
7465 tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7466 key->domain, error);
7468 return -rte_errno; /* No need to refill the error info */
7469 tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7470 /* Lookup from cache. */
7471 LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7472 if (matcher->crc == cache_matcher->crc &&
7473 matcher->priority == cache_matcher->priority &&
7474 !memcmp((const void *)matcher->mask.buf,
7475 (const void *)cache_matcher->mask.buf,
7476 cache_matcher->mask.size)) {
7478 "%s group %u priority %hd use %s "
7479 "matcher %p: refcnt %d++",
7480 key->domain ? "FDB" : "NIC", key->table_id,
7481 cache_matcher->priority,
7482 key->direction ? "tx" : "rx",
7483 (void *)cache_matcher,
7484 rte_atomic32_read(&cache_matcher->refcnt));
7485 rte_atomic32_inc(&cache_matcher->refcnt);
7486 dev_flow->handle->dvh.matcher = cache_matcher;
7487 /* old matcher should not make the table ref++. */
7488 flow_dv_tbl_resource_release(dev, tbl);
7492 /* Register new matcher. */
7493 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
7494 if (!cache_matcher) {
7495 flow_dv_tbl_resource_release(dev, tbl);
7496 return rte_flow_error_set(error, ENOMEM,
7497 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7498 "cannot allocate matcher memory");
7500 *cache_matcher = *matcher;
7501 dv_attr.match_criteria_enable =
7502 flow_dv_matcher_enable(cache_matcher->mask.buf);
7503 dv_attr.priority = matcher->priority;
7505 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7506 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
7507 &cache_matcher->matcher_object);
7509 rte_free(cache_matcher);
7510 #ifdef HAVE_MLX5DV_DR
7511 flow_dv_tbl_resource_release(dev, tbl);
7513 return rte_flow_error_set(error, ENOMEM,
7514 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7515 NULL, "cannot create matcher");
7517 /* Save the table information */
7518 cache_matcher->tbl = tbl;
7519 rte_atomic32_init(&cache_matcher->refcnt);
7520 /* only matcher ref++, table ref++ already done above in get API. */
7521 rte_atomic32_inc(&cache_matcher->refcnt);
7522 LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7523 dev_flow->handle->dvh.matcher = cache_matcher;
7524 DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7525 key->domain ? "FDB" : "NIC", key->table_id,
7526 cache_matcher->priority,
7527 key->direction ? "tx" : "rx", (void *)cache_matcher,
7528 rte_atomic32_read(&cache_matcher->refcnt));
7533 * Find existing tag resource or create and register a new one.
7535 * @param dev[in, out]
7536 * Pointer to rte_eth_dev structure.
7537 * @param[in, out] tag_be24
7538 * Tag value in big endian then R-shift 8.
7539 * @parm[in, out] dev_flow
7540 * Pointer to the dev_flow.
7542 * pointer to error structure.
7545 * 0 on success otherwise -errno and errno is set.
7548 flow_dv_tag_resource_register
7549 (struct rte_eth_dev *dev,
7551 struct mlx5_flow *dev_flow,
7552 struct rte_flow_error *error)
7554 struct mlx5_priv *priv = dev->data->dev_private;
7555 struct mlx5_dev_ctx_shared *sh = priv->sh;
7556 struct mlx5_flow_dv_tag_resource *cache_resource;
7557 struct mlx5_hlist_entry *entry;
7560 /* Lookup a matching resource from cache. */
7561 entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7563 cache_resource = container_of
7564 (entry, struct mlx5_flow_dv_tag_resource, entry);
7565 rte_atomic32_inc(&cache_resource->refcnt);
7566 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7567 dev_flow->dv.tag_resource = cache_resource;
7568 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7569 (void *)cache_resource,
7570 rte_atomic32_read(&cache_resource->refcnt));
7573 /* Register new resource. */
7574 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7575 &dev_flow->handle->dvh.rix_tag);
7576 if (!cache_resource)
7577 return rte_flow_error_set(error, ENOMEM,
7578 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7579 "cannot allocate resource memory");
7580 cache_resource->entry.key = (uint64_t)tag_be24;
7581 ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
7582 &cache_resource->action);
7584 rte_free(cache_resource);
7585 return rte_flow_error_set(error, ENOMEM,
7586 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7587 NULL, "cannot create action");
7589 rte_atomic32_init(&cache_resource->refcnt);
7590 rte_atomic32_inc(&cache_resource->refcnt);
7591 if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7592 mlx5_flow_os_destroy_flow_action(cache_resource->action);
7593 rte_free(cache_resource);
7594 return rte_flow_error_set(error, EEXIST,
7595 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7596 NULL, "cannot insert tag");
7598 dev_flow->dv.tag_resource = cache_resource;
7599 DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7600 (void *)cache_resource,
7601 rte_atomic32_read(&cache_resource->refcnt));
7609 * Pointer to Ethernet device.
7614 * 1 while a reference on it exists, 0 when freed.
7617 flow_dv_tag_release(struct rte_eth_dev *dev,
7620 struct mlx5_priv *priv = dev->data->dev_private;
7621 struct mlx5_dev_ctx_shared *sh = priv->sh;
7622 struct mlx5_flow_dv_tag_resource *tag;
7624 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7627 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7628 dev->data->port_id, (void *)tag,
7629 rte_atomic32_read(&tag->refcnt));
7630 if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7631 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
7632 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7633 DRV_LOG(DEBUG, "port %u tag %p: removed",
7634 dev->data->port_id, (void *)tag);
7635 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7642 * Translate port ID action to vport.
7645 * Pointer to rte_eth_dev structure.
7647 * Pointer to the port ID action.
7648 * @param[out] dst_port_id
7649 * The target port ID.
7651 * Pointer to the error structure.
7654 * 0 on success, a negative errno value otherwise and rte_errno is set.
7657 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7658 const struct rte_flow_action *action,
7659 uint32_t *dst_port_id,
7660 struct rte_flow_error *error)
7663 struct mlx5_priv *priv;
7664 const struct rte_flow_action_port_id *conf =
7665 (const struct rte_flow_action_port_id *)action->conf;
7667 port = conf->original ? dev->data->port_id : conf->id;
7668 priv = mlx5_port_to_eswitch_info(port, false);
7670 return rte_flow_error_set(error, -rte_errno,
7671 RTE_FLOW_ERROR_TYPE_ACTION,
7673 "No eswitch info was found for port");
7674 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7676 * This parameter is transferred to
7677 * mlx5dv_dr_action_create_dest_ib_port().
7679 *dst_port_id = priv->dev_port;
7682 * Legacy mode, no LAG configurations is supported.
7683 * This parameter is transferred to
7684 * mlx5dv_dr_action_create_dest_vport().
7686 *dst_port_id = priv->vport_id;
7692 * Create a counter with aging configuration.
7695 * Pointer to rte_eth_dev structure.
7697 * Pointer to the counter action configuration.
7699 * Pointer to the aging action configuration.
7702 * Index to flow counter on success, 0 otherwise.
7705 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7706 struct mlx5_flow *dev_flow,
7707 const struct rte_flow_action_count *count,
7708 const struct rte_flow_action_age *age)
7711 struct mlx5_age_param *age_param;
7713 counter = flow_dv_counter_alloc(dev,
7714 count ? count->shared : 0,
7715 count ? count->id : 0,
7716 dev_flow->dv.group, !!age);
7717 if (!counter || age == NULL)
7719 age_param = flow_dv_counter_idx_get_age(dev, counter);
7721 * The counter age accuracy may have a bit delay. Have 3/4
7722 * second bias on the timeount in order to let it age in time.
7724 age_param->context = age->context ? age->context :
7725 (void *)(uintptr_t)(dev_flow->flow_idx);
7727 * The counter age accuracy may have a bit delay. Have 3/4
7728 * second bias on the timeount in order to let it age in time.
7730 age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7731 /* Set expire time in unit of 0.1 sec. */
7732 age_param->port_id = dev->data->port_id;
7733 age_param->expire = age_param->timeout +
7734 rte_rdtsc() / (rte_get_tsc_hz() / 10);
7735 rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7739 * Add Tx queue matcher
7742 * Pointer to the dev struct.
7743 * @param[in, out] matcher
7745 * @param[in, out] key
7746 * Flow matcher value.
7748 * Flow pattern to translate.
7750 * Item is inner pattern.
7753 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7754 void *matcher, void *key,
7755 const struct rte_flow_item *item)
7757 const struct mlx5_rte_flow_item_tx_queue *queue_m;
7758 const struct mlx5_rte_flow_item_tx_queue *queue_v;
7760 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7762 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7763 struct mlx5_txq_ctrl *txq;
7767 queue_m = (const void *)item->mask;
7770 queue_v = (const void *)item->spec;
7773 txq = mlx5_txq_get(dev, queue_v->queue);
7776 queue = txq->obj->sq->id;
7777 MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7778 MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7779 queue & queue_m->queue);
7780 mlx5_txq_release(dev, queue_v->queue);
7784 * Set the hash fields according to the @p flow information.
7786 * @param[in] dev_flow
7787 * Pointer to the mlx5_flow.
7788 * @param[in] rss_desc
7789 * Pointer to the mlx5_flow_rss_desc.
7792 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7793 struct mlx5_flow_rss_desc *rss_desc)
7795 uint64_t items = dev_flow->handle->layers;
7797 uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7799 dev_flow->hash_fields = 0;
7800 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7801 if (rss_desc->level >= 2) {
7802 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7806 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7807 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7808 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7809 if (rss_types & ETH_RSS_L3_SRC_ONLY)
7810 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7811 else if (rss_types & ETH_RSS_L3_DST_ONLY)
7812 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7814 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7816 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7817 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7818 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7819 if (rss_types & ETH_RSS_L3_SRC_ONLY)
7820 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7821 else if (rss_types & ETH_RSS_L3_DST_ONLY)
7822 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7824 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7827 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7828 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7829 if (rss_types & ETH_RSS_UDP) {
7830 if (rss_types & ETH_RSS_L4_SRC_ONLY)
7831 dev_flow->hash_fields |=
7832 IBV_RX_HASH_SRC_PORT_UDP;
7833 else if (rss_types & ETH_RSS_L4_DST_ONLY)
7834 dev_flow->hash_fields |=
7835 IBV_RX_HASH_DST_PORT_UDP;
7837 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7839 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7840 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7841 if (rss_types & ETH_RSS_TCP) {
7842 if (rss_types & ETH_RSS_L4_SRC_ONLY)
7843 dev_flow->hash_fields |=
7844 IBV_RX_HASH_SRC_PORT_TCP;
7845 else if (rss_types & ETH_RSS_L4_DST_ONLY)
7846 dev_flow->hash_fields |=
7847 IBV_RX_HASH_DST_PORT_TCP;
7849 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7855 * Fill the flow with DV spec, lock free
7856 * (mutex should be acquired by caller).
7859 * Pointer to rte_eth_dev structure.
7860 * @param[in, out] dev_flow
7861 * Pointer to the sub flow.
7863 * Pointer to the flow attributes.
7865 * Pointer to the list of items.
7866 * @param[in] actions
7867 * Pointer to the list of actions.
7869 * Pointer to the error structure.
7872 * 0 on success, a negative errno value otherwise and rte_errno is set.
7875 __flow_dv_translate(struct rte_eth_dev *dev,
7876 struct mlx5_flow *dev_flow,
7877 const struct rte_flow_attr *attr,
7878 const struct rte_flow_item items[],
7879 const struct rte_flow_action actions[],
7880 struct rte_flow_error *error)
7882 struct mlx5_priv *priv = dev->data->dev_private;
7883 struct mlx5_dev_config *dev_conf = &priv->config;
7884 struct rte_flow *flow = dev_flow->flow;
7885 struct mlx5_flow_handle *handle = dev_flow->handle;
7886 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7888 [!!priv->flow_nested_idx];
7889 uint64_t item_flags = 0;
7890 uint64_t last_item = 0;
7891 uint64_t action_flags = 0;
7892 uint64_t priority = attr->priority;
7893 struct mlx5_flow_dv_matcher matcher = {
7895 .size = sizeof(matcher.mask.buf),
7899 bool actions_end = false;
7901 struct mlx5_flow_dv_modify_hdr_resource res;
7902 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
7903 sizeof(struct mlx5_modification_cmd) *
7904 (MLX5_MAX_MODIFY_NUM + 1)];
7906 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
7907 const struct rte_flow_action_count *count = NULL;
7908 const struct rte_flow_action_age *age = NULL;
7909 union flow_dv_attr flow_attr = { .attr = 0 };
7911 union mlx5_flow_tbl_key tbl_key;
7912 uint32_t modify_action_position = UINT32_MAX;
7913 void *match_mask = matcher.mask.buf;
7914 void *match_value = dev_flow->dv.value.buf;
7915 uint8_t next_protocol = 0xff;
7916 struct rte_vlan_hdr vlan = { 0 };
7920 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
7921 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
7922 ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
7923 !!priv->fdb_def_rule, &table, error);
7926 dev_flow->dv.group = table;
7928 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
7929 if (priority == MLX5_FLOW_PRIO_RSVD)
7930 priority = dev_conf->flow_prio - 1;
7931 /* number of actions must be set to 0 in case of dirty stack. */
7932 mhdr_res->actions_num = 0;
7933 for (; !actions_end ; actions++) {
7934 const struct rte_flow_action_queue *queue;
7935 const struct rte_flow_action_rss *rss;
7936 const struct rte_flow_action *action = actions;
7937 const uint8_t *rss_key;
7938 const struct rte_flow_action_jump *jump_data;
7939 const struct rte_flow_action_meter *mtr;
7940 struct mlx5_flow_tbl_resource *tbl;
7941 uint32_t port_id = 0;
7942 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
7943 int action_type = actions->type;
7944 const struct rte_flow_action *found_action = NULL;
7945 struct mlx5_flow_meter *fm = NULL;
7947 if (!mlx5_flow_os_action_supported(action_type))
7948 return rte_flow_error_set(error, ENOTSUP,
7949 RTE_FLOW_ERROR_TYPE_ACTION,
7951 "action not supported");
7952 switch (action_type) {
7953 case RTE_FLOW_ACTION_TYPE_VOID:
7955 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7956 if (flow_dv_translate_action_port_id(dev, action,
7959 port_id_resource.port_id = port_id;
7960 MLX5_ASSERT(!handle->rix_port_id_action);
7961 if (flow_dv_port_id_action_resource_register
7962 (dev, &port_id_resource, dev_flow, error))
7964 dev_flow->dv.actions[actions_n++] =
7965 dev_flow->dv.port_id_action->action;
7966 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7967 dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
7969 case RTE_FLOW_ACTION_TYPE_FLAG:
7970 action_flags |= MLX5_FLOW_ACTION_FLAG;
7971 dev_flow->handle->mark = 1;
7972 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7973 struct rte_flow_action_mark mark = {
7974 .id = MLX5_FLOW_MARK_DEFAULT,
7977 if (flow_dv_convert_action_mark(dev, &mark,
7981 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7984 tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
7986 * Only one FLAG or MARK is supported per device flow
7987 * right now. So the pointer to the tag resource must be
7988 * zero before the register process.
7990 MLX5_ASSERT(!handle->dvh.rix_tag);
7991 if (flow_dv_tag_resource_register(dev, tag_be,
7994 MLX5_ASSERT(dev_flow->dv.tag_resource);
7995 dev_flow->dv.actions[actions_n++] =
7996 dev_flow->dv.tag_resource->action;
7998 case RTE_FLOW_ACTION_TYPE_MARK:
7999 action_flags |= MLX5_FLOW_ACTION_MARK;
8000 dev_flow->handle->mark = 1;
8001 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8002 const struct rte_flow_action_mark *mark =
8003 (const struct rte_flow_action_mark *)
8006 if (flow_dv_convert_action_mark(dev, mark,
8010 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8014 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
8015 /* Legacy (non-extensive) MARK action. */
8016 tag_be = mlx5_flow_mark_set
8017 (((const struct rte_flow_action_mark *)
8018 (actions->conf))->id);
8019 MLX5_ASSERT(!handle->dvh.rix_tag);
8020 if (flow_dv_tag_resource_register(dev, tag_be,
8023 MLX5_ASSERT(dev_flow->dv.tag_resource);
8024 dev_flow->dv.actions[actions_n++] =
8025 dev_flow->dv.tag_resource->action;
8027 case RTE_FLOW_ACTION_TYPE_SET_META:
8028 if (flow_dv_convert_action_set_meta
8029 (dev, mhdr_res, attr,
8030 (const struct rte_flow_action_set_meta *)
8031 actions->conf, error))
8033 action_flags |= MLX5_FLOW_ACTION_SET_META;
8035 case RTE_FLOW_ACTION_TYPE_SET_TAG:
8036 if (flow_dv_convert_action_set_tag
8038 (const struct rte_flow_action_set_tag *)
8039 actions->conf, error))
8041 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8043 case RTE_FLOW_ACTION_TYPE_DROP:
8044 action_flags |= MLX5_FLOW_ACTION_DROP;
8045 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8047 case RTE_FLOW_ACTION_TYPE_QUEUE:
8048 queue = actions->conf;
8049 rss_desc->queue_num = 1;
8050 rss_desc->queue[0] = queue->index;
8051 action_flags |= MLX5_FLOW_ACTION_QUEUE;
8052 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8054 case RTE_FLOW_ACTION_TYPE_RSS:
8055 rss = actions->conf;
8056 memcpy(rss_desc->queue, rss->queue,
8057 rss->queue_num * sizeof(uint16_t));
8058 rss_desc->queue_num = rss->queue_num;
8059 /* NULL RSS key indicates default RSS key. */
8060 rss_key = !rss->key ? rss_hash_default_key : rss->key;
8061 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8063 * rss->level and rss.types should be set in advance
8064 * when expanding items for RSS.
8066 action_flags |= MLX5_FLOW_ACTION_RSS;
8067 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8069 case RTE_FLOW_ACTION_TYPE_AGE:
8070 case RTE_FLOW_ACTION_TYPE_COUNT:
8071 if (!dev_conf->devx) {
8072 return rte_flow_error_set
8074 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8076 "count action not supported");
8078 /* Save information first, will apply later. */
8079 if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8080 count = action->conf;
8083 action_flags |= MLX5_FLOW_ACTION_COUNT;
8085 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8086 dev_flow->dv.actions[actions_n++] =
8087 priv->sh->pop_vlan_action;
8088 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8090 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8091 if (!(action_flags &
8092 MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8093 flow_dev_get_vlan_info_from_items(items, &vlan);
8094 vlan.eth_proto = rte_be_to_cpu_16
8095 ((((const struct rte_flow_action_of_push_vlan *)
8096 actions->conf)->ethertype));
8097 found_action = mlx5_flow_find_action
8099 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8101 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8102 found_action = mlx5_flow_find_action
8104 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8106 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8107 if (flow_dv_create_action_push_vlan
8108 (dev, attr, &vlan, dev_flow, error))
8110 dev_flow->dv.actions[actions_n++] =
8111 dev_flow->dv.push_vlan_res->action;
8112 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8114 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8115 /* of_vlan_push action handled this action */
8116 MLX5_ASSERT(action_flags &
8117 MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8119 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8120 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8122 flow_dev_get_vlan_info_from_items(items, &vlan);
8123 mlx5_update_vlan_vid_pcp(actions, &vlan);
8124 /* If no VLAN push - this is a modify header action */
8125 if (flow_dv_convert_action_modify_vlan_vid
8126 (mhdr_res, actions, error))
8128 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8130 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8131 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8132 if (flow_dv_create_action_l2_encap(dev, actions,
8137 dev_flow->dv.actions[actions_n++] =
8138 dev_flow->dv.encap_decap->action;
8139 action_flags |= MLX5_FLOW_ACTION_ENCAP;
8141 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8142 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8143 if (flow_dv_create_action_l2_decap(dev, dev_flow,
8147 dev_flow->dv.actions[actions_n++] =
8148 dev_flow->dv.encap_decap->action;
8149 action_flags |= MLX5_FLOW_ACTION_DECAP;
8151 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8152 /* Handle encap with preceding decap. */
8153 if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8154 if (flow_dv_create_action_raw_encap
8155 (dev, actions, dev_flow, attr, error))
8157 dev_flow->dv.actions[actions_n++] =
8158 dev_flow->dv.encap_decap->action;
8160 /* Handle encap without preceding decap. */
8161 if (flow_dv_create_action_l2_encap
8162 (dev, actions, dev_flow, attr->transfer,
8165 dev_flow->dv.actions[actions_n++] =
8166 dev_flow->dv.encap_decap->action;
8168 action_flags |= MLX5_FLOW_ACTION_ENCAP;
8170 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8171 while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8173 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8174 if (flow_dv_create_action_l2_decap
8175 (dev, dev_flow, attr->transfer, error))
8177 dev_flow->dv.actions[actions_n++] =
8178 dev_flow->dv.encap_decap->action;
8180 /* If decap is followed by encap, handle it at encap. */
8181 action_flags |= MLX5_FLOW_ACTION_DECAP;
8183 case RTE_FLOW_ACTION_TYPE_JUMP:
8184 jump_data = action->conf;
8185 ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8187 !!priv->fdb_def_rule,
8191 tbl = flow_dv_tbl_resource_get(dev, table,
8193 attr->transfer, error);
8195 return rte_flow_error_set
8197 RTE_FLOW_ERROR_TYPE_ACTION,
8199 "cannot create jump action.");
8200 if (flow_dv_jump_tbl_resource_register
8201 (dev, tbl, dev_flow, error)) {
8202 flow_dv_tbl_resource_release(dev, tbl);
8203 return rte_flow_error_set
8205 RTE_FLOW_ERROR_TYPE_ACTION,
8207 "cannot create jump action.");
8209 dev_flow->dv.actions[actions_n++] =
8210 dev_flow->dv.jump->action;
8211 action_flags |= MLX5_FLOW_ACTION_JUMP;
8212 dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8214 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8215 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8216 if (flow_dv_convert_action_modify_mac
8217 (mhdr_res, actions, error))
8219 action_flags |= actions->type ==
8220 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8221 MLX5_FLOW_ACTION_SET_MAC_SRC :
8222 MLX5_FLOW_ACTION_SET_MAC_DST;
8224 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8225 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8226 if (flow_dv_convert_action_modify_ipv4
8227 (mhdr_res, actions, error))
8229 action_flags |= actions->type ==
8230 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8231 MLX5_FLOW_ACTION_SET_IPV4_SRC :
8232 MLX5_FLOW_ACTION_SET_IPV4_DST;
8234 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8235 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8236 if (flow_dv_convert_action_modify_ipv6
8237 (mhdr_res, actions, error))
8239 action_flags |= actions->type ==
8240 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8241 MLX5_FLOW_ACTION_SET_IPV6_SRC :
8242 MLX5_FLOW_ACTION_SET_IPV6_DST;
8244 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8245 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8246 if (flow_dv_convert_action_modify_tp
8247 (mhdr_res, actions, items,
8248 &flow_attr, dev_flow, !!(action_flags &
8249 MLX5_FLOW_ACTION_DECAP), error))
8251 action_flags |= actions->type ==
8252 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8253 MLX5_FLOW_ACTION_SET_TP_SRC :
8254 MLX5_FLOW_ACTION_SET_TP_DST;
8256 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8257 if (flow_dv_convert_action_modify_dec_ttl
8258 (mhdr_res, items, &flow_attr, dev_flow,
8260 MLX5_FLOW_ACTION_DECAP), error))
8262 action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8264 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8265 if (flow_dv_convert_action_modify_ttl
8266 (mhdr_res, actions, items, &flow_attr,
8267 dev_flow, !!(action_flags &
8268 MLX5_FLOW_ACTION_DECAP), error))
8270 action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8272 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8273 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8274 if (flow_dv_convert_action_modify_tcp_seq
8275 (mhdr_res, actions, error))
8277 action_flags |= actions->type ==
8278 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8279 MLX5_FLOW_ACTION_INC_TCP_SEQ :
8280 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8283 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8284 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8285 if (flow_dv_convert_action_modify_tcp_ack
8286 (mhdr_res, actions, error))
8288 action_flags |= actions->type ==
8289 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8290 MLX5_FLOW_ACTION_INC_TCP_ACK :
8291 MLX5_FLOW_ACTION_DEC_TCP_ACK;
8293 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8294 if (flow_dv_convert_action_set_reg
8295 (mhdr_res, actions, error))
8297 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8299 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8300 if (flow_dv_convert_action_copy_mreg
8301 (dev, mhdr_res, actions, error))
8303 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8305 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8306 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8307 dev_flow->handle->fate_action =
8308 MLX5_FLOW_FATE_DEFAULT_MISS;
8310 case RTE_FLOW_ACTION_TYPE_METER:
8311 mtr = actions->conf;
8313 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8316 return rte_flow_error_set(error,
8318 RTE_FLOW_ERROR_TYPE_ACTION,
8321 "or invalid parameters");
8322 flow->meter = fm->idx;
8324 /* Set the meter action. */
8326 fm = mlx5_ipool_get(priv->sh->ipool
8327 [MLX5_IPOOL_MTR], flow->meter);
8329 return rte_flow_error_set(error,
8331 RTE_FLOW_ERROR_TYPE_ACTION,
8334 "or invalid parameters");
8336 dev_flow->dv.actions[actions_n++] =
8337 fm->mfts->meter_action;
8338 action_flags |= MLX5_FLOW_ACTION_METER;
8340 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8341 if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8344 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8346 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8347 if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8350 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8352 case RTE_FLOW_ACTION_TYPE_END:
8354 if (mhdr_res->actions_num) {
8355 /* create modify action if needed. */
8356 if (flow_dv_modify_hdr_resource_register
8357 (dev, mhdr_res, dev_flow, error))
8359 dev_flow->dv.actions[modify_action_position] =
8360 handle->dvh.modify_hdr->action;
8362 if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8364 flow_dv_translate_create_counter(dev,
8365 dev_flow, count, age);
8368 return rte_flow_error_set
8370 RTE_FLOW_ERROR_TYPE_ACTION,
8372 "cannot create counter"
8374 dev_flow->dv.actions[actions_n++] =
8375 (flow_dv_counter_get_by_idx(dev,
8376 flow->counter, NULL))->action;
8382 if (mhdr_res->actions_num &&
8383 modify_action_position == UINT32_MAX)
8384 modify_action_position = actions_n++;
8386 dev_flow->dv.actions_n = actions_n;
8387 dev_flow->act_flags = action_flags;
8388 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8389 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8390 int item_type = items->type;
8392 if (!mlx5_flow_os_item_supported(item_type))
8393 return rte_flow_error_set(error, ENOTSUP,
8394 RTE_FLOW_ERROR_TYPE_ITEM,
8395 NULL, "item not supported");
8396 switch (item_type) {
8397 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8398 flow_dv_translate_item_port_id(dev, match_mask,
8399 match_value, items);
8400 last_item = MLX5_FLOW_ITEM_PORT_ID;
8402 case RTE_FLOW_ITEM_TYPE_ETH:
8403 flow_dv_translate_item_eth(match_mask, match_value,
8405 dev_flow->dv.group);
8406 matcher.priority = action_flags &
8407 MLX5_FLOW_ACTION_DEFAULT_MISS &&
8408 !dev_flow->external ?
8409 MLX5_PRIORITY_MAP_L3 :
8410 MLX5_PRIORITY_MAP_L2;
8411 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8412 MLX5_FLOW_LAYER_OUTER_L2;
8414 case RTE_FLOW_ITEM_TYPE_VLAN:
8415 flow_dv_translate_item_vlan(dev_flow,
8416 match_mask, match_value,
8418 dev_flow->dv.group);
8419 matcher.priority = MLX5_PRIORITY_MAP_L2;
8420 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8421 MLX5_FLOW_LAYER_INNER_VLAN) :
8422 (MLX5_FLOW_LAYER_OUTER_L2 |
8423 MLX5_FLOW_LAYER_OUTER_VLAN);
8425 case RTE_FLOW_ITEM_TYPE_IPV4:
8426 mlx5_flow_tunnel_ip_check(items, next_protocol,
8427 &item_flags, &tunnel);
8428 flow_dv_translate_item_ipv4(match_mask, match_value,
8429 items, item_flags, tunnel,
8430 dev_flow->dv.group);
8431 matcher.priority = MLX5_PRIORITY_MAP_L3;
8432 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8433 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8434 if (items->mask != NULL &&
8435 ((const struct rte_flow_item_ipv4 *)
8436 items->mask)->hdr.next_proto_id) {
8438 ((const struct rte_flow_item_ipv4 *)
8439 (items->spec))->hdr.next_proto_id;
8441 ((const struct rte_flow_item_ipv4 *)
8442 (items->mask))->hdr.next_proto_id;
8444 /* Reset for inner layer. */
8445 next_protocol = 0xff;
8448 case RTE_FLOW_ITEM_TYPE_IPV6:
8449 mlx5_flow_tunnel_ip_check(items, next_protocol,
8450 &item_flags, &tunnel);
8451 flow_dv_translate_item_ipv6(match_mask, match_value,
8452 items, item_flags, tunnel,
8453 dev_flow->dv.group);
8454 matcher.priority = MLX5_PRIORITY_MAP_L3;
8455 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8456 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8457 if (items->mask != NULL &&
8458 ((const struct rte_flow_item_ipv6 *)
8459 items->mask)->hdr.proto) {
8461 ((const struct rte_flow_item_ipv6 *)
8462 items->spec)->hdr.proto;
8464 ((const struct rte_flow_item_ipv6 *)
8465 items->mask)->hdr.proto;
8467 /* Reset for inner layer. */
8468 next_protocol = 0xff;
8471 case RTE_FLOW_ITEM_TYPE_TCP:
8472 flow_dv_translate_item_tcp(match_mask, match_value,
8474 matcher.priority = MLX5_PRIORITY_MAP_L4;
8475 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8476 MLX5_FLOW_LAYER_OUTER_L4_TCP;
8478 case RTE_FLOW_ITEM_TYPE_UDP:
8479 flow_dv_translate_item_udp(match_mask, match_value,
8481 matcher.priority = MLX5_PRIORITY_MAP_L4;
8482 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8483 MLX5_FLOW_LAYER_OUTER_L4_UDP;
8485 case RTE_FLOW_ITEM_TYPE_GRE:
8486 flow_dv_translate_item_gre(match_mask, match_value,
8488 matcher.priority = rss_desc->level >= 2 ?
8489 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8490 last_item = MLX5_FLOW_LAYER_GRE;
8492 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8493 flow_dv_translate_item_gre_key(match_mask,
8494 match_value, items);
8495 last_item = MLX5_FLOW_LAYER_GRE_KEY;
8497 case RTE_FLOW_ITEM_TYPE_NVGRE:
8498 flow_dv_translate_item_nvgre(match_mask, match_value,
8500 matcher.priority = rss_desc->level >= 2 ?
8501 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8502 last_item = MLX5_FLOW_LAYER_GRE;
8504 case RTE_FLOW_ITEM_TYPE_VXLAN:
8505 flow_dv_translate_item_vxlan(match_mask, match_value,
8507 matcher.priority = rss_desc->level >= 2 ?
8508 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8509 last_item = MLX5_FLOW_LAYER_VXLAN;
8511 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8512 flow_dv_translate_item_vxlan_gpe(match_mask,
8515 matcher.priority = rss_desc->level >= 2 ?
8516 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8517 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8519 case RTE_FLOW_ITEM_TYPE_GENEVE:
8520 flow_dv_translate_item_geneve(match_mask, match_value,
8522 matcher.priority = rss_desc->level >= 2 ?
8523 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8524 last_item = MLX5_FLOW_LAYER_GENEVE;
8526 case RTE_FLOW_ITEM_TYPE_MPLS:
8527 flow_dv_translate_item_mpls(match_mask, match_value,
8528 items, last_item, tunnel);
8529 matcher.priority = rss_desc->level >= 2 ?
8530 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8531 last_item = MLX5_FLOW_LAYER_MPLS;
8533 case RTE_FLOW_ITEM_TYPE_MARK:
8534 flow_dv_translate_item_mark(dev, match_mask,
8535 match_value, items);
8536 last_item = MLX5_FLOW_ITEM_MARK;
8538 case RTE_FLOW_ITEM_TYPE_META:
8539 flow_dv_translate_item_meta(dev, match_mask,
8540 match_value, attr, items);
8541 last_item = MLX5_FLOW_ITEM_METADATA;
8543 case RTE_FLOW_ITEM_TYPE_ICMP:
8544 flow_dv_translate_item_icmp(match_mask, match_value,
8546 last_item = MLX5_FLOW_LAYER_ICMP;
8548 case RTE_FLOW_ITEM_TYPE_ICMP6:
8549 flow_dv_translate_item_icmp6(match_mask, match_value,
8551 last_item = MLX5_FLOW_LAYER_ICMP6;
8553 case RTE_FLOW_ITEM_TYPE_TAG:
8554 flow_dv_translate_item_tag(dev, match_mask,
8555 match_value, items);
8556 last_item = MLX5_FLOW_ITEM_TAG;
8558 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8559 flow_dv_translate_mlx5_item_tag(dev, match_mask,
8560 match_value, items);
8561 last_item = MLX5_FLOW_ITEM_TAG;
8563 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8564 flow_dv_translate_item_tx_queue(dev, match_mask,
8567 last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8569 case RTE_FLOW_ITEM_TYPE_GTP:
8570 flow_dv_translate_item_gtp(match_mask, match_value,
8572 matcher.priority = rss_desc->level >= 2 ?
8573 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8574 last_item = MLX5_FLOW_LAYER_GTP;
8579 item_flags |= last_item;
8582 * When E-Switch mode is enabled, we have two cases where we need to
8583 * set the source port manually.
8584 * The first one, is in case of Nic steering rule, and the second is
8585 * E-Switch rule where no port_id item was found. In both cases
8586 * the source port is set according the current port in use.
8588 if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8589 (priv->representor || priv->master)) {
8590 if (flow_dv_translate_item_port_id(dev, match_mask,
8594 #ifdef RTE_LIBRTE_MLX5_DEBUG
8595 MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8596 dev_flow->dv.value.buf));
8599 * Layers may be already initialized from prefix flow if this dev_flow
8600 * is the suffix flow.
8602 handle->layers |= item_flags;
8603 if (action_flags & MLX5_FLOW_ACTION_RSS)
8604 flow_dv_hashfields_set(dev_flow, rss_desc);
8605 /* Register matcher. */
8606 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8608 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8610 /* reserved field no needs to be set to 0 here. */
8611 tbl_key.domain = attr->transfer;
8612 tbl_key.direction = attr->egress;
8613 tbl_key.table_id = dev_flow->dv.group;
8614 if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8620 * Apply the flow to the NIC, lock free,
8621 * (mutex should be acquired by caller).
8624 * Pointer to the Ethernet device structure.
8625 * @param[in, out] flow
8626 * Pointer to flow structure.
8628 * Pointer to error structure.
8631 * 0 on success, a negative errno value otherwise and rte_errno is set.
8634 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8635 struct rte_flow_error *error)
8637 struct mlx5_flow_dv_workspace *dv;
8638 struct mlx5_flow_handle *dh;
8639 struct mlx5_flow_handle_dv *dv_h;
8640 struct mlx5_flow *dev_flow;
8641 struct mlx5_priv *priv = dev->data->dev_private;
8642 uint32_t handle_idx;
8647 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8648 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8650 dh = dev_flow->handle;
8653 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8655 dv->actions[n++] = priv->sh->esw_drop_action;
8657 struct mlx5_hrxq *drop_hrxq;
8658 drop_hrxq = mlx5_hrxq_drop_new(dev);
8662 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8664 "cannot get drop hash queue");
8668 * Drop queues will be released by the specify
8669 * mlx5_hrxq_drop_release() function. Assign
8670 * the special index to hrxq to mark the queue
8671 * has been allocated.
8673 dh->rix_hrxq = UINT32_MAX;
8674 dv->actions[n++] = drop_hrxq->action;
8676 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8677 struct mlx5_hrxq *hrxq;
8679 struct mlx5_flow_rss_desc *rss_desc =
8680 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8681 [!!priv->flow_nested_idx];
8683 MLX5_ASSERT(rss_desc->queue_num);
8684 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8685 MLX5_RSS_HASH_KEY_LEN,
8686 dev_flow->hash_fields,
8688 rss_desc->queue_num);
8690 hrxq_idx = mlx5_hrxq_new
8691 (dev, rss_desc->key,
8692 MLX5_RSS_HASH_KEY_LEN,
8693 dev_flow->hash_fields,
8695 rss_desc->queue_num,
8697 MLX5_FLOW_LAYER_TUNNEL));
8699 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8704 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8705 "cannot get hash queue");
8708 dh->rix_hrxq = hrxq_idx;
8709 dv->actions[n++] = hrxq->action;
8710 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
8711 if (flow_dv_default_miss_resource_register
8715 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8716 "cannot create default miss resource");
8717 goto error_default_miss;
8719 dh->rix_default_fate = MLX5_FLOW_FATE_DEFAULT_MISS;
8720 dv->actions[n++] = priv->sh->default_miss.action;
8722 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
8723 (void *)&dv->value, n,
8724 dv->actions, &dh->drv_flow);
8726 rte_flow_error_set(error, errno,
8727 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8729 "hardware refuses to create flow");
8732 if (priv->vmwa_context &&
8733 dh->vf_vlan.tag && !dh->vf_vlan.created) {
8735 * The rule contains the VLAN pattern.
8736 * For VF we are going to create VLAN
8737 * interface to make hypervisor set correct
8738 * e-Switch vport context.
8740 mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8745 if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
8746 flow_dv_default_miss_resource_release(dev);
8748 err = rte_errno; /* Save rte_errno before cleanup. */
8749 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8750 handle_idx, dh, next) {
8751 /* hrxq is union, don't clear it if the flag is not set. */
8753 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8754 mlx5_hrxq_drop_release(dev);
8756 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8757 mlx5_hrxq_release(dev, dh->rix_hrxq);
8761 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8762 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8764 rte_errno = err; /* Restore rte_errno. */
8769 * Release the flow matcher.
8772 * Pointer to Ethernet device.
8774 * Pointer to mlx5_flow_handle.
8777 * 1 while a reference on it exists, 0 when freed.
8780 flow_dv_matcher_release(struct rte_eth_dev *dev,
8781 struct mlx5_flow_handle *handle)
8783 struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8785 MLX5_ASSERT(matcher->matcher_object);
8786 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8787 dev->data->port_id, (void *)matcher,
8788 rte_atomic32_read(&matcher->refcnt));
8789 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8790 claim_zero(mlx5_flow_os_destroy_flow_matcher
8791 (matcher->matcher_object));
8792 LIST_REMOVE(matcher, next);
8793 /* table ref-- in release interface. */
8794 flow_dv_tbl_resource_release(dev, matcher->tbl);
8796 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8797 dev->data->port_id, (void *)matcher);
8804 * Release an encap/decap resource.
8807 * Pointer to Ethernet device.
8809 * Pointer to mlx5_flow_handle.
8812 * 1 while a reference on it exists, 0 when freed.
8815 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8816 struct mlx5_flow_handle *handle)
8818 struct mlx5_priv *priv = dev->data->dev_private;
8819 uint32_t idx = handle->dvh.rix_encap_decap;
8820 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8822 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8824 if (!cache_resource)
8826 MLX5_ASSERT(cache_resource->action);
8827 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8828 (void *)cache_resource,
8829 rte_atomic32_read(&cache_resource->refcnt));
8830 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8831 claim_zero(mlx5_flow_os_destroy_flow_action
8832 (cache_resource->action));
8833 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8834 &priv->sh->encaps_decaps, idx,
8835 cache_resource, next);
8836 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8837 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8838 (void *)cache_resource);
8845 * Release an jump to table action resource.
8848 * Pointer to Ethernet device.
8850 * Pointer to mlx5_flow_handle.
8853 * 1 while a reference on it exists, 0 when freed.
8856 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8857 struct mlx5_flow_handle *handle)
8859 struct mlx5_priv *priv = dev->data->dev_private;
8860 struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8861 struct mlx5_flow_tbl_data_entry *tbl_data;
8863 tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8867 cache_resource = &tbl_data->jump;
8868 MLX5_ASSERT(cache_resource->action);
8869 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
8870 (void *)cache_resource,
8871 rte_atomic32_read(&cache_resource->refcnt));
8872 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8873 claim_zero(mlx5_flow_os_destroy_flow_action
8874 (cache_resource->action));
8875 /* jump action memory free is inside the table release. */
8876 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
8877 DRV_LOG(DEBUG, "jump table resource %p: removed",
8878 (void *)cache_resource);
8885 * Release a default miss resource.
8888 * Pointer to Ethernet device.
8890 * 1 while a reference on it exists, 0 when freed.
8893 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
8895 struct mlx5_priv *priv = dev->data->dev_private;
8896 struct mlx5_dev_ctx_shared *sh = priv->sh;
8897 struct mlx5_flow_default_miss_resource *cache_resource =
8900 MLX5_ASSERT(cache_resource->action);
8901 DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
8902 (void *)cache_resource->action,
8903 rte_atomic32_read(&cache_resource->refcnt));
8904 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8905 claim_zero(mlx5_glue->destroy_flow_action
8906 (cache_resource->action));
8907 DRV_LOG(DEBUG, "default miss resource %p: removed",
8908 (void *)cache_resource->action);
8915 * Release a modify-header resource.
8918 * Pointer to mlx5_flow_handle.
8921 * 1 while a reference on it exists, 0 when freed.
8924 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
8926 struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
8927 handle->dvh.modify_hdr;
8929 MLX5_ASSERT(cache_resource->action);
8930 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
8931 (void *)cache_resource,
8932 rte_atomic32_read(&cache_resource->refcnt));
8933 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8934 claim_zero(mlx5_flow_os_destroy_flow_action
8935 (cache_resource->action));
8936 LIST_REMOVE(cache_resource, next);
8937 rte_free(cache_resource);
8938 DRV_LOG(DEBUG, "modify-header resource %p: removed",
8939 (void *)cache_resource);
8946 * Release port ID action resource.
8949 * Pointer to Ethernet device.
8951 * Pointer to mlx5_flow_handle.
8954 * 1 while a reference on it exists, 0 when freed.
8957 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
8958 struct mlx5_flow_handle *handle)
8960 struct mlx5_priv *priv = dev->data->dev_private;
8961 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
8962 uint32_t idx = handle->rix_port_id_action;
8964 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8966 if (!cache_resource)
8968 MLX5_ASSERT(cache_resource->action);
8969 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
8970 (void *)cache_resource,
8971 rte_atomic32_read(&cache_resource->refcnt));
8972 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8973 claim_zero(mlx5_flow_os_destroy_flow_action
8974 (cache_resource->action));
8975 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8976 &priv->sh->port_id_action_list, idx,
8977 cache_resource, next);
8978 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
8979 DRV_LOG(DEBUG, "port id action resource %p: removed",
8980 (void *)cache_resource);
8987 * Release push vlan action resource.
8990 * Pointer to Ethernet device.
8992 * Pointer to mlx5_flow_handle.
8995 * 1 while a reference on it exists, 0 when freed.
8998 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
8999 struct mlx5_flow_handle *handle)
9001 struct mlx5_priv *priv = dev->data->dev_private;
9002 uint32_t idx = handle->dvh.rix_push_vlan;
9003 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
9005 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9007 if (!cache_resource)
9009 MLX5_ASSERT(cache_resource->action);
9010 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
9011 (void *)cache_resource,
9012 rte_atomic32_read(&cache_resource->refcnt));
9013 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9014 claim_zero(mlx5_flow_os_destroy_flow_action
9015 (cache_resource->action));
9016 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9017 &priv->sh->push_vlan_action_list, idx,
9018 cache_resource, next);
9019 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
9020 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
9021 (void *)cache_resource);
9028 * Release the fate resource.
9031 * Pointer to Ethernet device.
9033 * Pointer to mlx5_flow_handle.
9036 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
9037 struct mlx5_flow_handle *handle)
9039 if (!handle->rix_fate)
9041 switch (handle->fate_action) {
9042 case MLX5_FLOW_FATE_DROP:
9043 mlx5_hrxq_drop_release(dev);
9045 case MLX5_FLOW_FATE_QUEUE:
9046 mlx5_hrxq_release(dev, handle->rix_hrxq);
9048 case MLX5_FLOW_FATE_JUMP:
9049 flow_dv_jump_tbl_resource_release(dev, handle);
9051 case MLX5_FLOW_FATE_PORT_ID:
9052 flow_dv_port_id_action_resource_release(dev, handle);
9054 case MLX5_FLOW_FATE_DEFAULT_MISS:
9055 flow_dv_default_miss_resource_release(dev);
9058 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9061 handle->rix_fate = 0;
9065 * Remove the flow from the NIC but keeps it in memory.
9066 * Lock free, (mutex should be acquired by caller).
9069 * Pointer to Ethernet device.
9070 * @param[in, out] flow
9071 * Pointer to flow structure.
9074 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9076 struct mlx5_flow_handle *dh;
9077 uint32_t handle_idx;
9078 struct mlx5_priv *priv = dev->data->dev_private;
9082 handle_idx = flow->dev_handles;
9083 while (handle_idx) {
9084 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9089 claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
9090 dh->drv_flow = NULL;
9092 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9093 dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9094 dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9095 flow_dv_fate_resource_release(dev, dh);
9096 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9097 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9098 handle_idx = dh->next.next;
9103 * Remove the flow from the NIC and the memory.
9104 * Lock free, (mutex should be acquired by caller).
9107 * Pointer to the Ethernet device structure.
9108 * @param[in, out] flow
9109 * Pointer to flow structure.
9112 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9114 struct mlx5_flow_handle *dev_handle;
9115 struct mlx5_priv *priv = dev->data->dev_private;
9119 __flow_dv_remove(dev, flow);
9120 if (flow->counter) {
9121 flow_dv_counter_release(dev, flow->counter);
9125 struct mlx5_flow_meter *fm;
9127 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9130 mlx5_flow_meter_detach(fm);
9133 while (flow->dev_handles) {
9134 uint32_t tmp_idx = flow->dev_handles;
9136 dev_handle = mlx5_ipool_get(priv->sh->ipool
9137 [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9140 flow->dev_handles = dev_handle->next.next;
9141 if (dev_handle->dvh.matcher)
9142 flow_dv_matcher_release(dev, dev_handle);
9143 if (dev_handle->dvh.rix_encap_decap)
9144 flow_dv_encap_decap_resource_release(dev, dev_handle);
9145 if (dev_handle->dvh.modify_hdr)
9146 flow_dv_modify_hdr_resource_release(dev_handle);
9147 if (dev_handle->dvh.rix_push_vlan)
9148 flow_dv_push_vlan_action_resource_release(dev,
9150 if (dev_handle->dvh.rix_tag)
9151 flow_dv_tag_release(dev,
9152 dev_handle->dvh.rix_tag);
9153 flow_dv_fate_resource_release(dev, dev_handle);
9154 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9160 * Query a dv flow rule for its statistics via devx.
9163 * Pointer to Ethernet device.
9165 * Pointer to the sub flow.
9167 * data retrieved by the query.
9169 * Perform verbose error reporting if not NULL.
9172 * 0 on success, a negative errno value otherwise and rte_errno is set.
9175 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9176 void *data, struct rte_flow_error *error)
9178 struct mlx5_priv *priv = dev->data->dev_private;
9179 struct rte_flow_query_count *qc = data;
9181 if (!priv->config.devx)
9182 return rte_flow_error_set(error, ENOTSUP,
9183 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9185 "counters are not supported");
9186 if (flow->counter) {
9187 uint64_t pkts, bytes;
9188 struct mlx5_flow_counter *cnt;
9190 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9192 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9196 return rte_flow_error_set(error, -err,
9197 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9198 NULL, "cannot read counters");
9201 qc->hits = pkts - cnt->hits;
9202 qc->bytes = bytes - cnt->bytes;
9209 return rte_flow_error_set(error, EINVAL,
9210 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9212 "counters are not available");
9218 * @see rte_flow_query()
9222 flow_dv_query(struct rte_eth_dev *dev,
9223 struct rte_flow *flow __rte_unused,
9224 const struct rte_flow_action *actions __rte_unused,
9225 void *data __rte_unused,
9226 struct rte_flow_error *error __rte_unused)
9230 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9231 switch (actions->type) {
9232 case RTE_FLOW_ACTION_TYPE_VOID:
9234 case RTE_FLOW_ACTION_TYPE_COUNT:
9235 ret = flow_dv_query_count(dev, flow, data, error);
9238 return rte_flow_error_set(error, ENOTSUP,
9239 RTE_FLOW_ERROR_TYPE_ACTION,
9241 "action not supported");
9248 * Destroy the meter table set.
9249 * Lock free, (mutex should be acquired by caller).
9252 * Pointer to Ethernet device.
9254 * Pointer to the meter table set.
9260 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9261 struct mlx5_meter_domains_infos *tbl)
9263 struct mlx5_priv *priv = dev->data->dev_private;
9264 struct mlx5_meter_domains_infos *mtd =
9265 (struct mlx5_meter_domains_infos *)tbl;
9267 if (!mtd || !priv->config.dv_flow_en)
9269 if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9270 claim_zero(mlx5_flow_os_destroy_flow
9271 (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9272 if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9273 claim_zero(mlx5_flow_os_destroy_flow
9274 (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9275 if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9276 claim_zero(mlx5_flow_os_destroy_flow
9277 (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9278 if (mtd->egress.color_matcher)
9279 claim_zero(mlx5_flow_os_destroy_flow_matcher
9280 (mtd->egress.color_matcher));
9281 if (mtd->egress.any_matcher)
9282 claim_zero(mlx5_flow_os_destroy_flow_matcher
9283 (mtd->egress.any_matcher));
9284 if (mtd->egress.tbl)
9285 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9286 if (mtd->egress.sfx_tbl)
9287 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9288 if (mtd->ingress.color_matcher)
9289 claim_zero(mlx5_flow_os_destroy_flow_matcher
9290 (mtd->ingress.color_matcher));
9291 if (mtd->ingress.any_matcher)
9292 claim_zero(mlx5_flow_os_destroy_flow_matcher
9293 (mtd->ingress.any_matcher));
9294 if (mtd->ingress.tbl)
9295 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9296 if (mtd->ingress.sfx_tbl)
9297 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9298 if (mtd->transfer.color_matcher)
9299 claim_zero(mlx5_flow_os_destroy_flow_matcher
9300 (mtd->transfer.color_matcher));
9301 if (mtd->transfer.any_matcher)
9302 claim_zero(mlx5_flow_os_destroy_flow_matcher
9303 (mtd->transfer.any_matcher));
9304 if (mtd->transfer.tbl)
9305 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9306 if (mtd->transfer.sfx_tbl)
9307 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9309 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
9314 /* Number of meter flow actions, count and jump or count and drop. */
9315 #define METER_ACTIONS 2
9318 * Create specify domain meter table and suffix table.
9321 * Pointer to Ethernet device.
9322 * @param[in,out] mtb
9323 * Pointer to DV meter table set.
9326 * @param[in] transfer
9328 * @param[in] color_reg_c_idx
9329 * Reg C index for color match.
9332 * 0 on success, -1 otherwise and rte_errno is set.
9335 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9336 struct mlx5_meter_domains_infos *mtb,
9337 uint8_t egress, uint8_t transfer,
9338 uint32_t color_reg_c_idx)
9340 struct mlx5_priv *priv = dev->data->dev_private;
9341 struct mlx5_dev_ctx_shared *sh = priv->sh;
9342 struct mlx5_flow_dv_match_params mask = {
9343 .size = sizeof(mask.buf),
9345 struct mlx5_flow_dv_match_params value = {
9346 .size = sizeof(value.buf),
9348 struct mlx5dv_flow_matcher_attr dv_attr = {
9349 .type = IBV_FLOW_ATTR_NORMAL,
9351 .match_criteria_enable = 0,
9352 .match_mask = (void *)&mask,
9354 void *actions[METER_ACTIONS];
9355 struct mlx5_meter_domain_info *dtb;
9356 struct rte_flow_error error;
9361 dtb = &mtb->transfer;
9365 dtb = &mtb->ingress;
9366 /* Create the meter table with METER level. */
9367 dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9368 egress, transfer, &error);
9370 DRV_LOG(ERR, "Failed to create meter policer table.");
9373 /* Create the meter suffix table with SUFFIX level. */
9374 dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9375 MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9376 egress, transfer, &error);
9377 if (!dtb->sfx_tbl) {
9378 DRV_LOG(ERR, "Failed to create meter suffix table.");
9381 /* Create matchers, Any and Color. */
9382 dv_attr.priority = 3;
9383 dv_attr.match_criteria_enable = 0;
9384 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9387 DRV_LOG(ERR, "Failed to create meter"
9388 " policer default matcher.");
9391 dv_attr.priority = 0;
9392 dv_attr.match_criteria_enable =
9393 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9394 flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9395 rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9396 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9397 &dtb->color_matcher);
9399 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9402 if (mtb->count_actns[RTE_MTR_DROPPED])
9403 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9404 actions[i++] = mtb->drop_actn;
9405 /* Default rule: lowest priority, match any, actions: drop. */
9406 ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
9408 &dtb->policer_rules[RTE_MTR_DROPPED]);
9410 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9419 * Create the needed meter and suffix tables.
9420 * Lock free, (mutex should be acquired by caller).
9423 * Pointer to Ethernet device.
9425 * Pointer to the flow meter.
9428 * Pointer to table set on success, NULL otherwise and rte_errno is set.
9430 static struct mlx5_meter_domains_infos *
9431 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9432 const struct mlx5_flow_meter *fm)
9434 struct mlx5_priv *priv = dev->data->dev_private;
9435 struct mlx5_meter_domains_infos *mtb;
9439 if (!priv->mtr_en) {
9440 rte_errno = ENOTSUP;
9443 mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0);
9445 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9448 /* Create meter count actions */
9449 for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9450 struct mlx5_flow_counter *cnt;
9451 if (!fm->policer_stats.cnt[i])
9453 cnt = flow_dv_counter_get_by_idx(dev,
9454 fm->policer_stats.cnt[i], NULL);
9455 mtb->count_actns[i] = cnt->action;
9457 /* Create drop action. */
9458 ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
9460 DRV_LOG(ERR, "Failed to create drop action.");
9463 /* Egress meter table. */
9464 ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9466 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9469 /* Ingress meter table. */
9470 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9472 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9475 /* FDB meter table. */
9476 if (priv->config.dv_esw_en) {
9477 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9478 priv->mtr_color_reg);
9480 DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9486 flow_dv_destroy_mtr_tbl(dev, mtb);
9491 * Destroy domain policer rule.
9494 * Pointer to domain table.
9497 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9501 for (i = 0; i < RTE_MTR_DROPPED; i++) {
9502 if (dt->policer_rules[i]) {
9503 claim_zero(mlx5_flow_os_destroy_flow
9504 (dt->policer_rules[i]));
9505 dt->policer_rules[i] = NULL;
9508 if (dt->jump_actn) {
9509 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
9510 dt->jump_actn = NULL;
9515 * Destroy policer rules.
9518 * Pointer to Ethernet device.
9520 * Pointer to flow meter structure.
9522 * Pointer to flow attributes.
9528 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9529 const struct mlx5_flow_meter *fm,
9530 const struct rte_flow_attr *attr)
9532 struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9537 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9539 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9541 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9546 * Create specify domain meter policer rule.
9549 * Pointer to flow meter structure.
9551 * Pointer to DV meter table set.
9552 * @param[in] mtr_reg_c
9553 * Color match REG_C.
9556 * 0 on success, -1 otherwise.
9559 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9560 struct mlx5_meter_domain_info *dtb,
9563 struct mlx5_flow_dv_match_params matcher = {
9564 .size = sizeof(matcher.buf),
9566 struct mlx5_flow_dv_match_params value = {
9567 .size = sizeof(value.buf),
9569 struct mlx5_meter_domains_infos *mtb = fm->mfts;
9570 void *actions[METER_ACTIONS];
9574 /* Create jump action. */
9575 if (!dtb->jump_actn)
9576 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
9577 (dtb->sfx_tbl->obj, &dtb->jump_actn);
9579 DRV_LOG(ERR, "Failed to create policer jump action.");
9582 for (i = 0; i < RTE_MTR_DROPPED; i++) {
9585 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9586 rte_col_2_mlx5_col(i), UINT8_MAX);
9587 if (mtb->count_actns[i])
9588 actions[j++] = mtb->count_actns[i];
9589 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9590 actions[j++] = mtb->drop_actn;
9592 actions[j++] = dtb->jump_actn;
9593 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
9594 (void *)&value, j, actions,
9595 &dtb->policer_rules[i]);
9597 DRV_LOG(ERR, "Failed to create policer rule.");
9608 * Create policer rules.
9611 * Pointer to Ethernet device.
9613 * Pointer to flow meter structure.
9615 * Pointer to flow attributes.
9618 * 0 on success, -1 otherwise.
9621 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9622 struct mlx5_flow_meter *fm,
9623 const struct rte_flow_attr *attr)
9625 struct mlx5_priv *priv = dev->data->dev_private;
9626 struct mlx5_meter_domains_infos *mtb = fm->mfts;
9630 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9631 priv->mtr_color_reg);
9633 DRV_LOG(ERR, "Failed to create egress policer.");
9637 if (attr->ingress) {
9638 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9639 priv->mtr_color_reg);
9641 DRV_LOG(ERR, "Failed to create ingress policer.");
9645 if (attr->transfer) {
9646 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9647 priv->mtr_color_reg);
9649 DRV_LOG(ERR, "Failed to create transfer policer.");
9655 flow_dv_destroy_policer_rules(dev, fm, attr);
9660 * Query a devx counter.
9663 * Pointer to the Ethernet device structure.
9665 * Index to the flow counter.
9667 * Set to clear the counter statistics.
9669 * The statistics value of packets.
9671 * The statistics value of bytes.
9674 * 0 on success, otherwise return -1.
9677 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9678 uint64_t *pkts, uint64_t *bytes)
9680 struct mlx5_priv *priv = dev->data->dev_private;
9681 struct mlx5_flow_counter *cnt;
9682 uint64_t inn_pkts, inn_bytes;
9685 if (!priv->config.devx)
9688 ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9691 cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9692 *pkts = inn_pkts - cnt->hits;
9693 *bytes = inn_bytes - cnt->bytes;
9695 cnt->hits = inn_pkts;
9696 cnt->bytes = inn_bytes;
9702 * Get aged-out flows.
9705 * Pointer to the Ethernet device structure.
9706 * @param[in] context
9707 * The address of an array of pointers to the aged-out flows contexts.
9708 * @param[in] nb_contexts
9709 * The length of context array pointers.
9711 * Perform verbose error reporting if not NULL. Initialized in case of
9715 * how many contexts get in success, otherwise negative errno value.
9716 * if nb_contexts is 0, return the amount of all aged contexts.
9717 * if nb_contexts is not 0 , return the amount of aged flows reported
9718 * in the context array.
9719 * @note: only stub for now
9722 flow_get_aged_flows(struct rte_eth_dev *dev,
9724 uint32_t nb_contexts,
9725 struct rte_flow_error *error)
9727 struct mlx5_priv *priv = dev->data->dev_private;
9728 struct mlx5_age_info *age_info;
9729 struct mlx5_age_param *age_param;
9730 struct mlx5_flow_counter *counter;
9733 if (nb_contexts && !context)
9734 return rte_flow_error_set(error, EINVAL,
9735 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9737 "Should assign at least one flow or"
9738 " context to get if nb_contexts != 0");
9739 age_info = GET_PORT_AGE_INFO(priv);
9740 rte_spinlock_lock(&age_info->aged_sl);
9741 TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9744 age_param = MLX5_CNT_TO_AGE(counter);
9745 context[nb_flows - 1] = age_param->context;
9746 if (!(--nb_contexts))
9750 rte_spinlock_unlock(&age_info->aged_sl);
9751 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9756 * Mutex-protected thunk to lock-free __flow_dv_translate().
9759 flow_dv_translate(struct rte_eth_dev *dev,
9760 struct mlx5_flow *dev_flow,
9761 const struct rte_flow_attr *attr,
9762 const struct rte_flow_item items[],
9763 const struct rte_flow_action actions[],
9764 struct rte_flow_error *error)
9768 flow_dv_shared_lock(dev);
9769 ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9770 flow_dv_shared_unlock(dev);
9775 * Mutex-protected thunk to lock-free __flow_dv_apply().
9778 flow_dv_apply(struct rte_eth_dev *dev,
9779 struct rte_flow *flow,
9780 struct rte_flow_error *error)
9784 flow_dv_shared_lock(dev);
9785 ret = __flow_dv_apply(dev, flow, error);
9786 flow_dv_shared_unlock(dev);
9791 * Mutex-protected thunk to lock-free __flow_dv_remove().
9794 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9796 flow_dv_shared_lock(dev);
9797 __flow_dv_remove(dev, flow);
9798 flow_dv_shared_unlock(dev);
9802 * Mutex-protected thunk to lock-free __flow_dv_destroy().
9805 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9807 flow_dv_shared_lock(dev);
9808 __flow_dv_destroy(dev, flow);
9809 flow_dv_shared_unlock(dev);
9813 * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9816 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9820 flow_dv_shared_lock(dev);
9821 cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9822 flow_dv_shared_unlock(dev);
9827 * Mutex-protected thunk to lock-free flow_dv_counter_release().
9830 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9832 flow_dv_shared_lock(dev);
9833 flow_dv_counter_release(dev, cnt);
9834 flow_dv_shared_unlock(dev);
9837 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9838 .validate = flow_dv_validate,
9839 .prepare = flow_dv_prepare,
9840 .translate = flow_dv_translate,
9841 .apply = flow_dv_apply,
9842 .remove = flow_dv_remove,
9843 .destroy = flow_dv_destroy,
9844 .query = flow_dv_query,
9845 .create_mtr_tbls = flow_dv_create_mtr_tbl,
9846 .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9847 .create_policer_rules = flow_dv_create_policer_rules,
9848 .destroy_policer_rules = flow_dv_destroy_policer_rules,
9849 .counter_alloc = flow_dv_counter_allocate,
9850 .counter_free = flow_dv_counter_free,
9851 .counter_query = flow_dv_counter_query,
9852 .get_aged_flows = flow_get_aged_flows,
9855 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */