1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
11 #include <rte_common.h>
12 #include <rte_ether.h>
13 #include <rte_ethdev_driver.h>
15 #include <rte_flow_driver.h>
16 #include <rte_malloc.h>
17 #include <rte_cycles.h>
20 #include <rte_vxlan.h>
22 #include <rte_eal_paging.h>
25 #include <mlx5_glue.h>
26 #include <mlx5_devx_cmds.h>
28 #include <mlx5_malloc.h>
30 #include "mlx5_defs.h"
32 #include "mlx5_common_os.h"
33 #include "mlx5_flow.h"
34 #include "mlx5_flow_os.h"
35 #include "mlx5_rxtx.h"
37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
43 #ifndef HAVE_MLX5DV_DR_ESWITCH
44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
49 #ifndef HAVE_MLX5DV_DR
50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
53 /* VLAN header definitions */
54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
74 struct mlx5_flow_tbl_resource *tbl);
77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
80 * Initialize flow attributes structure according to flow items' types.
82 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
83 * mode. For tunnel mode, the items to be modified are the outermost ones.
86 * Pointer to item specification.
88 * Pointer to flow attributes structure.
90 * Pointer to the sub flow.
91 * @param[in] tunnel_decap
92 * Whether action is after tunnel decapsulation.
95 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
96 struct mlx5_flow *dev_flow, bool tunnel_decap)
98 uint64_t layers = dev_flow->handle->layers;
101 * If layers is already initialized, it means this dev_flow is the
102 * suffix flow, the layers flags is set by the prefix flow. Need to
103 * use the layer flags from prefix flow as the suffix flow may not
104 * have the user defined items as the flow is split.
107 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
109 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
111 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
113 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
118 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
119 uint8_t next_protocol = 0xff;
120 switch (item->type) {
121 case RTE_FLOW_ITEM_TYPE_GRE:
122 case RTE_FLOW_ITEM_TYPE_NVGRE:
123 case RTE_FLOW_ITEM_TYPE_VXLAN:
124 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
125 case RTE_FLOW_ITEM_TYPE_GENEVE:
126 case RTE_FLOW_ITEM_TYPE_MPLS:
130 case RTE_FLOW_ITEM_TYPE_IPV4:
133 if (item->mask != NULL &&
134 ((const struct rte_flow_item_ipv4 *)
135 item->mask)->hdr.next_proto_id)
137 ((const struct rte_flow_item_ipv4 *)
138 (item->spec))->hdr.next_proto_id &
139 ((const struct rte_flow_item_ipv4 *)
140 (item->mask))->hdr.next_proto_id;
141 if ((next_protocol == IPPROTO_IPIP ||
142 next_protocol == IPPROTO_IPV6) && tunnel_decap)
145 case RTE_FLOW_ITEM_TYPE_IPV6:
148 if (item->mask != NULL &&
149 ((const struct rte_flow_item_ipv6 *)
150 item->mask)->hdr.proto)
152 ((const struct rte_flow_item_ipv6 *)
153 (item->spec))->hdr.proto &
154 ((const struct rte_flow_item_ipv6 *)
155 (item->mask))->hdr.proto;
156 if ((next_protocol == IPPROTO_IPIP ||
157 next_protocol == IPPROTO_IPV6) && tunnel_decap)
160 case RTE_FLOW_ITEM_TYPE_UDP:
164 case RTE_FLOW_ITEM_TYPE_TCP:
176 * Convert rte_mtr_color to mlx5 color.
185 rte_col_2_mlx5_col(enum rte_color rcol)
188 case RTE_COLOR_GREEN:
189 return MLX5_FLOW_COLOR_GREEN;
190 case RTE_COLOR_YELLOW:
191 return MLX5_FLOW_COLOR_YELLOW;
193 return MLX5_FLOW_COLOR_RED;
197 return MLX5_FLOW_COLOR_UNDEFINED;
200 struct field_modify_info {
201 uint32_t size; /* Size of field in protocol header, in bytes. */
202 uint32_t offset; /* Offset of field in protocol header, in bytes. */
203 enum mlx5_modification_field id;
206 struct field_modify_info modify_eth[] = {
207 {4, 0, MLX5_MODI_OUT_DMAC_47_16},
208 {2, 4, MLX5_MODI_OUT_DMAC_15_0},
209 {4, 6, MLX5_MODI_OUT_SMAC_47_16},
210 {2, 10, MLX5_MODI_OUT_SMAC_15_0},
214 struct field_modify_info modify_vlan_out_first_vid[] = {
215 /* Size in bits !!! */
216 {12, 0, MLX5_MODI_OUT_FIRST_VID},
220 struct field_modify_info modify_ipv4[] = {
221 {1, 1, MLX5_MODI_OUT_IP_DSCP},
222 {1, 8, MLX5_MODI_OUT_IPV4_TTL},
223 {4, 12, MLX5_MODI_OUT_SIPV4},
224 {4, 16, MLX5_MODI_OUT_DIPV4},
228 struct field_modify_info modify_ipv6[] = {
229 {1, 0, MLX5_MODI_OUT_IP_DSCP},
230 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
231 {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
232 {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
233 {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
234 {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
235 {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
236 {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
237 {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
238 {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
242 struct field_modify_info modify_udp[] = {
243 {2, 0, MLX5_MODI_OUT_UDP_SPORT},
244 {2, 2, MLX5_MODI_OUT_UDP_DPORT},
248 struct field_modify_info modify_tcp[] = {
249 {2, 0, MLX5_MODI_OUT_TCP_SPORT},
250 {2, 2, MLX5_MODI_OUT_TCP_DPORT},
251 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
252 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
257 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
258 uint8_t next_protocol, uint64_t *item_flags,
261 MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
262 item->type == RTE_FLOW_ITEM_TYPE_IPV6);
263 if (next_protocol == IPPROTO_IPIP) {
264 *item_flags |= MLX5_FLOW_LAYER_IPIP;
267 if (next_protocol == IPPROTO_IPV6) {
268 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
274 * Acquire the synchronizing object to protect multithreaded access
275 * to shared dv context. Lock occurs only if context is actually
276 * shared, i.e. we have multiport IB device and representors are
280 * Pointer to the rte_eth_dev structure.
283 flow_dv_shared_lock(struct rte_eth_dev *dev)
285 struct mlx5_priv *priv = dev->data->dev_private;
286 struct mlx5_dev_ctx_shared *sh = priv->sh;
288 if (sh->dv_refcnt > 1) {
291 ret = pthread_mutex_lock(&sh->dv_mutex);
298 flow_dv_shared_unlock(struct rte_eth_dev *dev)
300 struct mlx5_priv *priv = dev->data->dev_private;
301 struct mlx5_dev_ctx_shared *sh = priv->sh;
303 if (sh->dv_refcnt > 1) {
306 ret = pthread_mutex_unlock(&sh->dv_mutex);
312 /* Update VLAN's VID/PCP based on input rte_flow_action.
315 * Pointer to struct rte_flow_action.
317 * Pointer to struct rte_vlan_hdr.
320 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
321 struct rte_vlan_hdr *vlan)
324 if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
326 ((const struct rte_flow_action_of_set_vlan_pcp *)
327 action->conf)->vlan_pcp;
328 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
329 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
330 vlan->vlan_tci |= vlan_tci;
331 } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
332 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
333 vlan->vlan_tci |= rte_be_to_cpu_16
334 (((const struct rte_flow_action_of_set_vlan_vid *)
335 action->conf)->vlan_vid);
340 * Fetch 1, 2, 3 or 4 byte field from the byte array
341 * and return as unsigned integer in host-endian format.
344 * Pointer to data array.
346 * Size of field to extract.
349 * converted field in host endian format.
351 static inline uint32_t
352 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
361 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
364 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
365 ret = (ret << 8) | *(data + sizeof(uint16_t));
368 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
379 * Convert modify-header action to DV specification.
381 * Data length of each action is determined by provided field description
382 * and the item mask. Data bit offset and width of each action is determined
383 * by provided item mask.
386 * Pointer to item specification.
388 * Pointer to field modification information.
389 * For MLX5_MODIFICATION_TYPE_SET specifies destination field.
390 * For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
391 * For MLX5_MODIFICATION_TYPE_COPY specifies source field.
393 * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
394 * Negative offset value sets the same offset as source offset.
395 * size field is ignored, value is taken from source field.
396 * @param[in,out] resource
397 * Pointer to the modify-header resource.
399 * Type of modification.
401 * Pointer to the error structure.
404 * 0 on success, a negative errno value otherwise and rte_errno is set.
407 flow_dv_convert_modify_action(struct rte_flow_item *item,
408 struct field_modify_info *field,
409 struct field_modify_info *dcopy,
410 struct mlx5_flow_dv_modify_hdr_resource *resource,
411 uint32_t type, struct rte_flow_error *error)
413 uint32_t i = resource->actions_num;
414 struct mlx5_modification_cmd *actions = resource->actions;
417 * The item and mask are provided in big-endian format.
418 * The fields should be presented as in big-endian format either.
419 * Mask must be always present, it defines the actual field width.
421 MLX5_ASSERT(item->mask);
422 MLX5_ASSERT(field->size);
429 if (i >= MLX5_MAX_MODIFY_NUM)
430 return rte_flow_error_set(error, EINVAL,
431 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
432 "too many items to modify");
433 /* Fetch variable byte size mask from the array. */
434 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
435 field->offset, field->size);
440 /* Deduce actual data width in bits from mask value. */
441 off_b = rte_bsf32(mask);
442 size_b = sizeof(uint32_t) * CHAR_BIT -
443 off_b - __builtin_clz(mask);
445 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
446 actions[i] = (struct mlx5_modification_cmd) {
452 /* Convert entire record to expected big-endian format. */
453 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
454 if (type == MLX5_MODIFICATION_TYPE_COPY) {
456 actions[i].dst_field = dcopy->id;
457 actions[i].dst_offset =
458 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
459 /* Convert entire record to big-endian format. */
460 actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
462 MLX5_ASSERT(item->spec);
463 data = flow_dv_fetch_field((const uint8_t *)item->spec +
464 field->offset, field->size);
465 /* Shift out the trailing masked bits from data. */
466 data = (data & mask) >> off_b;
467 actions[i].data1 = rte_cpu_to_be_32(data);
471 } while (field->size);
472 if (resource->actions_num == i)
473 return rte_flow_error_set(error, EINVAL,
474 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
475 "invalid modification flow item");
476 resource->actions_num = i;
481 * Convert modify-header set IPv4 address action to DV specification.
483 * @param[in,out] resource
484 * Pointer to the modify-header resource.
486 * Pointer to action specification.
488 * Pointer to the error structure.
491 * 0 on success, a negative errno value otherwise and rte_errno is set.
494 flow_dv_convert_action_modify_ipv4
495 (struct mlx5_flow_dv_modify_hdr_resource *resource,
496 const struct rte_flow_action *action,
497 struct rte_flow_error *error)
499 const struct rte_flow_action_set_ipv4 *conf =
500 (const struct rte_flow_action_set_ipv4 *)(action->conf);
501 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
502 struct rte_flow_item_ipv4 ipv4;
503 struct rte_flow_item_ipv4 ipv4_mask;
505 memset(&ipv4, 0, sizeof(ipv4));
506 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
507 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
508 ipv4.hdr.src_addr = conf->ipv4_addr;
509 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
511 ipv4.hdr.dst_addr = conf->ipv4_addr;
512 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
515 item.mask = &ipv4_mask;
516 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
517 MLX5_MODIFICATION_TYPE_SET, error);
521 * Convert modify-header set IPv6 address action to DV specification.
523 * @param[in,out] resource
524 * Pointer to the modify-header resource.
526 * Pointer to action specification.
528 * Pointer to the error structure.
531 * 0 on success, a negative errno value otherwise and rte_errno is set.
534 flow_dv_convert_action_modify_ipv6
535 (struct mlx5_flow_dv_modify_hdr_resource *resource,
536 const struct rte_flow_action *action,
537 struct rte_flow_error *error)
539 const struct rte_flow_action_set_ipv6 *conf =
540 (const struct rte_flow_action_set_ipv6 *)(action->conf);
541 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
542 struct rte_flow_item_ipv6 ipv6;
543 struct rte_flow_item_ipv6 ipv6_mask;
545 memset(&ipv6, 0, sizeof(ipv6));
546 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
547 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
548 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
549 sizeof(ipv6.hdr.src_addr));
550 memcpy(&ipv6_mask.hdr.src_addr,
551 &rte_flow_item_ipv6_mask.hdr.src_addr,
552 sizeof(ipv6.hdr.src_addr));
554 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
555 sizeof(ipv6.hdr.dst_addr));
556 memcpy(&ipv6_mask.hdr.dst_addr,
557 &rte_flow_item_ipv6_mask.hdr.dst_addr,
558 sizeof(ipv6.hdr.dst_addr));
561 item.mask = &ipv6_mask;
562 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
563 MLX5_MODIFICATION_TYPE_SET, error);
567 * Convert modify-header set MAC address action to DV specification.
569 * @param[in,out] resource
570 * Pointer to the modify-header resource.
572 * Pointer to action specification.
574 * Pointer to the error structure.
577 * 0 on success, a negative errno value otherwise and rte_errno is set.
580 flow_dv_convert_action_modify_mac
581 (struct mlx5_flow_dv_modify_hdr_resource *resource,
582 const struct rte_flow_action *action,
583 struct rte_flow_error *error)
585 const struct rte_flow_action_set_mac *conf =
586 (const struct rte_flow_action_set_mac *)(action->conf);
587 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
588 struct rte_flow_item_eth eth;
589 struct rte_flow_item_eth eth_mask;
591 memset(ð, 0, sizeof(eth));
592 memset(ð_mask, 0, sizeof(eth_mask));
593 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
594 memcpy(ð.src.addr_bytes, &conf->mac_addr,
595 sizeof(eth.src.addr_bytes));
596 memcpy(ð_mask.src.addr_bytes,
597 &rte_flow_item_eth_mask.src.addr_bytes,
598 sizeof(eth_mask.src.addr_bytes));
600 memcpy(ð.dst.addr_bytes, &conf->mac_addr,
601 sizeof(eth.dst.addr_bytes));
602 memcpy(ð_mask.dst.addr_bytes,
603 &rte_flow_item_eth_mask.dst.addr_bytes,
604 sizeof(eth_mask.dst.addr_bytes));
607 item.mask = ð_mask;
608 return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
609 MLX5_MODIFICATION_TYPE_SET, error);
613 * Convert modify-header set VLAN VID action to DV specification.
615 * @param[in,out] resource
616 * Pointer to the modify-header resource.
618 * Pointer to action specification.
620 * Pointer to the error structure.
623 * 0 on success, a negative errno value otherwise and rte_errno is set.
626 flow_dv_convert_action_modify_vlan_vid
627 (struct mlx5_flow_dv_modify_hdr_resource *resource,
628 const struct rte_flow_action *action,
629 struct rte_flow_error *error)
631 const struct rte_flow_action_of_set_vlan_vid *conf =
632 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
633 int i = resource->actions_num;
634 struct mlx5_modification_cmd *actions = resource->actions;
635 struct field_modify_info *field = modify_vlan_out_first_vid;
637 if (i >= MLX5_MAX_MODIFY_NUM)
638 return rte_flow_error_set(error, EINVAL,
639 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
640 "too many items to modify");
641 actions[i] = (struct mlx5_modification_cmd) {
642 .action_type = MLX5_MODIFICATION_TYPE_SET,
644 .length = field->size,
645 .offset = field->offset,
647 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
648 actions[i].data1 = conf->vlan_vid;
649 actions[i].data1 = actions[i].data1 << 16;
650 resource->actions_num = ++i;
655 * Convert modify-header set TP action to DV specification.
657 * @param[in,out] resource
658 * Pointer to the modify-header resource.
660 * Pointer to action specification.
662 * Pointer to rte_flow_item objects list.
664 * Pointer to flow attributes structure.
665 * @param[in] dev_flow
666 * Pointer to the sub flow.
667 * @param[in] tunnel_decap
668 * Whether action is after tunnel decapsulation.
670 * Pointer to the error structure.
673 * 0 on success, a negative errno value otherwise and rte_errno is set.
676 flow_dv_convert_action_modify_tp
677 (struct mlx5_flow_dv_modify_hdr_resource *resource,
678 const struct rte_flow_action *action,
679 const struct rte_flow_item *items,
680 union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
681 bool tunnel_decap, struct rte_flow_error *error)
683 const struct rte_flow_action_set_tp *conf =
684 (const struct rte_flow_action_set_tp *)(action->conf);
685 struct rte_flow_item item;
686 struct rte_flow_item_udp udp;
687 struct rte_flow_item_udp udp_mask;
688 struct rte_flow_item_tcp tcp;
689 struct rte_flow_item_tcp tcp_mask;
690 struct field_modify_info *field;
693 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
695 memset(&udp, 0, sizeof(udp));
696 memset(&udp_mask, 0, sizeof(udp_mask));
697 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
698 udp.hdr.src_port = conf->port;
699 udp_mask.hdr.src_port =
700 rte_flow_item_udp_mask.hdr.src_port;
702 udp.hdr.dst_port = conf->port;
703 udp_mask.hdr.dst_port =
704 rte_flow_item_udp_mask.hdr.dst_port;
706 item.type = RTE_FLOW_ITEM_TYPE_UDP;
708 item.mask = &udp_mask;
711 MLX5_ASSERT(attr->tcp);
712 memset(&tcp, 0, sizeof(tcp));
713 memset(&tcp_mask, 0, sizeof(tcp_mask));
714 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
715 tcp.hdr.src_port = conf->port;
716 tcp_mask.hdr.src_port =
717 rte_flow_item_tcp_mask.hdr.src_port;
719 tcp.hdr.dst_port = conf->port;
720 tcp_mask.hdr.dst_port =
721 rte_flow_item_tcp_mask.hdr.dst_port;
723 item.type = RTE_FLOW_ITEM_TYPE_TCP;
725 item.mask = &tcp_mask;
728 return flow_dv_convert_modify_action(&item, field, NULL, resource,
729 MLX5_MODIFICATION_TYPE_SET, error);
733 * Convert modify-header set TTL action to DV specification.
735 * @param[in,out] resource
736 * Pointer to the modify-header resource.
738 * Pointer to action specification.
740 * Pointer to rte_flow_item objects list.
742 * Pointer to flow attributes structure.
743 * @param[in] dev_flow
744 * Pointer to the sub flow.
745 * @param[in] tunnel_decap
746 * Whether action is after tunnel decapsulation.
748 * Pointer to the error structure.
751 * 0 on success, a negative errno value otherwise and rte_errno is set.
754 flow_dv_convert_action_modify_ttl
755 (struct mlx5_flow_dv_modify_hdr_resource *resource,
756 const struct rte_flow_action *action,
757 const struct rte_flow_item *items,
758 union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
759 bool tunnel_decap, struct rte_flow_error *error)
761 const struct rte_flow_action_set_ttl *conf =
762 (const struct rte_flow_action_set_ttl *)(action->conf);
763 struct rte_flow_item item;
764 struct rte_flow_item_ipv4 ipv4;
765 struct rte_flow_item_ipv4 ipv4_mask;
766 struct rte_flow_item_ipv6 ipv6;
767 struct rte_flow_item_ipv6 ipv6_mask;
768 struct field_modify_info *field;
771 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
773 memset(&ipv4, 0, sizeof(ipv4));
774 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
775 ipv4.hdr.time_to_live = conf->ttl_value;
776 ipv4_mask.hdr.time_to_live = 0xFF;
777 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
779 item.mask = &ipv4_mask;
782 MLX5_ASSERT(attr->ipv6);
783 memset(&ipv6, 0, sizeof(ipv6));
784 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
785 ipv6.hdr.hop_limits = conf->ttl_value;
786 ipv6_mask.hdr.hop_limits = 0xFF;
787 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
789 item.mask = &ipv6_mask;
792 return flow_dv_convert_modify_action(&item, field, NULL, resource,
793 MLX5_MODIFICATION_TYPE_SET, error);
797 * Convert modify-header decrement TTL action to DV specification.
799 * @param[in,out] resource
800 * Pointer to the modify-header resource.
802 * Pointer to action specification.
804 * Pointer to rte_flow_item objects list.
806 * Pointer to flow attributes structure.
807 * @param[in] dev_flow
808 * Pointer to the sub flow.
809 * @param[in] tunnel_decap
810 * Whether action is after tunnel decapsulation.
812 * Pointer to the error structure.
815 * 0 on success, a negative errno value otherwise and rte_errno is set.
818 flow_dv_convert_action_modify_dec_ttl
819 (struct mlx5_flow_dv_modify_hdr_resource *resource,
820 const struct rte_flow_item *items,
821 union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
822 bool tunnel_decap, struct rte_flow_error *error)
824 struct rte_flow_item item;
825 struct rte_flow_item_ipv4 ipv4;
826 struct rte_flow_item_ipv4 ipv4_mask;
827 struct rte_flow_item_ipv6 ipv6;
828 struct rte_flow_item_ipv6 ipv6_mask;
829 struct field_modify_info *field;
832 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
834 memset(&ipv4, 0, sizeof(ipv4));
835 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
836 ipv4.hdr.time_to_live = 0xFF;
837 ipv4_mask.hdr.time_to_live = 0xFF;
838 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
840 item.mask = &ipv4_mask;
843 MLX5_ASSERT(attr->ipv6);
844 memset(&ipv6, 0, sizeof(ipv6));
845 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
846 ipv6.hdr.hop_limits = 0xFF;
847 ipv6_mask.hdr.hop_limits = 0xFF;
848 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
850 item.mask = &ipv6_mask;
853 return flow_dv_convert_modify_action(&item, field, NULL, resource,
854 MLX5_MODIFICATION_TYPE_ADD, error);
858 * Convert modify-header increment/decrement TCP Sequence number
859 * to DV specification.
861 * @param[in,out] resource
862 * Pointer to the modify-header resource.
864 * Pointer to action specification.
866 * Pointer to the error structure.
869 * 0 on success, a negative errno value otherwise and rte_errno is set.
872 flow_dv_convert_action_modify_tcp_seq
873 (struct mlx5_flow_dv_modify_hdr_resource *resource,
874 const struct rte_flow_action *action,
875 struct rte_flow_error *error)
877 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
878 uint64_t value = rte_be_to_cpu_32(*conf);
879 struct rte_flow_item item;
880 struct rte_flow_item_tcp tcp;
881 struct rte_flow_item_tcp tcp_mask;
883 memset(&tcp, 0, sizeof(tcp));
884 memset(&tcp_mask, 0, sizeof(tcp_mask));
885 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
887 * The HW has no decrement operation, only increment operation.
888 * To simulate decrement X from Y using increment operation
889 * we need to add UINT32_MAX X times to Y.
890 * Each adding of UINT32_MAX decrements Y by 1.
893 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
894 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
895 item.type = RTE_FLOW_ITEM_TYPE_TCP;
897 item.mask = &tcp_mask;
898 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
899 MLX5_MODIFICATION_TYPE_ADD, error);
903 * Convert modify-header increment/decrement TCP Acknowledgment number
904 * to DV specification.
906 * @param[in,out] resource
907 * Pointer to the modify-header resource.
909 * Pointer to action specification.
911 * Pointer to the error structure.
914 * 0 on success, a negative errno value otherwise and rte_errno is set.
917 flow_dv_convert_action_modify_tcp_ack
918 (struct mlx5_flow_dv_modify_hdr_resource *resource,
919 const struct rte_flow_action *action,
920 struct rte_flow_error *error)
922 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
923 uint64_t value = rte_be_to_cpu_32(*conf);
924 struct rte_flow_item item;
925 struct rte_flow_item_tcp tcp;
926 struct rte_flow_item_tcp tcp_mask;
928 memset(&tcp, 0, sizeof(tcp));
929 memset(&tcp_mask, 0, sizeof(tcp_mask));
930 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
932 * The HW has no decrement operation, only increment operation.
933 * To simulate decrement X from Y using increment operation
934 * we need to add UINT32_MAX X times to Y.
935 * Each adding of UINT32_MAX decrements Y by 1.
938 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
939 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
940 item.type = RTE_FLOW_ITEM_TYPE_TCP;
942 item.mask = &tcp_mask;
943 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
944 MLX5_MODIFICATION_TYPE_ADD, error);
947 static enum mlx5_modification_field reg_to_field[] = {
948 [REG_NON] = MLX5_MODI_OUT_NONE,
949 [REG_A] = MLX5_MODI_META_DATA_REG_A,
950 [REG_B] = MLX5_MODI_META_DATA_REG_B,
951 [REG_C_0] = MLX5_MODI_META_REG_C_0,
952 [REG_C_1] = MLX5_MODI_META_REG_C_1,
953 [REG_C_2] = MLX5_MODI_META_REG_C_2,
954 [REG_C_3] = MLX5_MODI_META_REG_C_3,
955 [REG_C_4] = MLX5_MODI_META_REG_C_4,
956 [REG_C_5] = MLX5_MODI_META_REG_C_5,
957 [REG_C_6] = MLX5_MODI_META_REG_C_6,
958 [REG_C_7] = MLX5_MODI_META_REG_C_7,
962 * Convert register set to DV specification.
964 * @param[in,out] resource
965 * Pointer to the modify-header resource.
967 * Pointer to action specification.
969 * Pointer to the error structure.
972 * 0 on success, a negative errno value otherwise and rte_errno is set.
975 flow_dv_convert_action_set_reg
976 (struct mlx5_flow_dv_modify_hdr_resource *resource,
977 const struct rte_flow_action *action,
978 struct rte_flow_error *error)
980 const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
981 struct mlx5_modification_cmd *actions = resource->actions;
982 uint32_t i = resource->actions_num;
984 if (i >= MLX5_MAX_MODIFY_NUM)
985 return rte_flow_error_set(error, EINVAL,
986 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
987 "too many items to modify");
988 MLX5_ASSERT(conf->id != REG_NON);
989 MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
990 actions[i] = (struct mlx5_modification_cmd) {
991 .action_type = MLX5_MODIFICATION_TYPE_SET,
992 .field = reg_to_field[conf->id],
994 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
995 actions[i].data1 = rte_cpu_to_be_32(conf->data);
997 resource->actions_num = i;
1002 * Convert SET_TAG action to DV specification.
1005 * Pointer to the rte_eth_dev structure.
1006 * @param[in,out] resource
1007 * Pointer to the modify-header resource.
1009 * Pointer to action specification.
1011 * Pointer to the error structure.
1014 * 0 on success, a negative errno value otherwise and rte_errno is set.
1017 flow_dv_convert_action_set_tag
1018 (struct rte_eth_dev *dev,
1019 struct mlx5_flow_dv_modify_hdr_resource *resource,
1020 const struct rte_flow_action_set_tag *conf,
1021 struct rte_flow_error *error)
1023 rte_be32_t data = rte_cpu_to_be_32(conf->data);
1024 rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1025 struct rte_flow_item item = {
1029 struct field_modify_info reg_c_x[] = {
1032 enum mlx5_modification_field reg_type;
1035 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1038 MLX5_ASSERT(ret != REG_NON);
1039 MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1040 reg_type = reg_to_field[ret];
1041 MLX5_ASSERT(reg_type > 0);
1042 reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1043 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1044 MLX5_MODIFICATION_TYPE_SET, error);
1048 * Convert internal COPY_REG action to DV specification.
1051 * Pointer to the rte_eth_dev structure.
1052 * @param[in,out] res
1053 * Pointer to the modify-header resource.
1055 * Pointer to action specification.
1057 * Pointer to the error structure.
1060 * 0 on success, a negative errno value otherwise and rte_errno is set.
1063 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1064 struct mlx5_flow_dv_modify_hdr_resource *res,
1065 const struct rte_flow_action *action,
1066 struct rte_flow_error *error)
1068 const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1069 rte_be32_t mask = RTE_BE32(UINT32_MAX);
1070 struct rte_flow_item item = {
1074 struct field_modify_info reg_src[] = {
1075 {4, 0, reg_to_field[conf->src]},
1078 struct field_modify_info reg_dst = {
1080 .id = reg_to_field[conf->dst],
1082 /* Adjust reg_c[0] usage according to reported mask. */
1083 if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1084 struct mlx5_priv *priv = dev->data->dev_private;
1085 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1087 MLX5_ASSERT(reg_c0);
1088 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1089 if (conf->dst == REG_C_0) {
1090 /* Copy to reg_c[0], within mask only. */
1091 reg_dst.offset = rte_bsf32(reg_c0);
1093 * Mask is ignoring the enianness, because
1094 * there is no conversion in datapath.
1096 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1097 /* Copy from destination lower bits to reg_c[0]. */
1098 mask = reg_c0 >> reg_dst.offset;
1100 /* Copy from destination upper bits to reg_c[0]. */
1101 mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1102 rte_fls_u32(reg_c0));
1105 mask = rte_cpu_to_be_32(reg_c0);
1106 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1107 /* Copy from reg_c[0] to destination lower bits. */
1110 /* Copy from reg_c[0] to destination upper bits. */
1111 reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1112 (rte_fls_u32(reg_c0) -
1117 return flow_dv_convert_modify_action(&item,
1118 reg_src, ®_dst, res,
1119 MLX5_MODIFICATION_TYPE_COPY,
1124 * Convert MARK action to DV specification. This routine is used
1125 * in extensive metadata only and requires metadata register to be
1126 * handled. In legacy mode hardware tag resource is engaged.
1129 * Pointer to the rte_eth_dev structure.
1131 * Pointer to MARK action specification.
1132 * @param[in,out] resource
1133 * Pointer to the modify-header resource.
1135 * Pointer to the error structure.
1138 * 0 on success, a negative errno value otherwise and rte_errno is set.
1141 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1142 const struct rte_flow_action_mark *conf,
1143 struct mlx5_flow_dv_modify_hdr_resource *resource,
1144 struct rte_flow_error *error)
1146 struct mlx5_priv *priv = dev->data->dev_private;
1147 rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1148 priv->sh->dv_mark_mask);
1149 rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1150 struct rte_flow_item item = {
1154 struct field_modify_info reg_c_x[] = {
1160 return rte_flow_error_set(error, EINVAL,
1161 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1162 NULL, "zero mark action mask");
1163 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1166 MLX5_ASSERT(reg > 0);
1167 if (reg == REG_C_0) {
1168 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1169 uint32_t shl_c0 = rte_bsf32(msk_c0);
1171 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1172 mask = rte_cpu_to_be_32(mask) & msk_c0;
1173 mask = rte_cpu_to_be_32(mask << shl_c0);
1175 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1176 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1177 MLX5_MODIFICATION_TYPE_SET, error);
1181 * Get metadata register index for specified steering domain.
1184 * Pointer to the rte_eth_dev structure.
1186 * Attributes of flow to determine steering domain.
1188 * Pointer to the error structure.
1191 * positive index on success, a negative errno value otherwise
1192 * and rte_errno is set.
1194 static enum modify_reg
1195 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1196 const struct rte_flow_attr *attr,
1197 struct rte_flow_error *error)
1200 mlx5_flow_get_reg_id(dev, attr->transfer ?
1204 MLX5_METADATA_RX, 0, error);
1206 return rte_flow_error_set(error,
1207 ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1208 NULL, "unavailable "
1209 "metadata register");
1214 * Convert SET_META action to DV specification.
1217 * Pointer to the rte_eth_dev structure.
1218 * @param[in,out] resource
1219 * Pointer to the modify-header resource.
1221 * Attributes of flow that includes this item.
1223 * Pointer to action specification.
1225 * Pointer to the error structure.
1228 * 0 on success, a negative errno value otherwise and rte_errno is set.
1231 flow_dv_convert_action_set_meta
1232 (struct rte_eth_dev *dev,
1233 struct mlx5_flow_dv_modify_hdr_resource *resource,
1234 const struct rte_flow_attr *attr,
1235 const struct rte_flow_action_set_meta *conf,
1236 struct rte_flow_error *error)
1238 uint32_t data = conf->data;
1239 uint32_t mask = conf->mask;
1240 struct rte_flow_item item = {
1244 struct field_modify_info reg_c_x[] = {
1247 int reg = flow_dv_get_metadata_reg(dev, attr, error);
1252 * In datapath code there is no endianness
1253 * coversions for perfromance reasons, all
1254 * pattern conversions are done in rte_flow.
1256 if (reg == REG_C_0) {
1257 struct mlx5_priv *priv = dev->data->dev_private;
1258 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1261 MLX5_ASSERT(msk_c0);
1262 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1263 shl_c0 = rte_bsf32(msk_c0);
1265 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1269 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1271 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1272 /* The routine expects parameters in memory as big-endian ones. */
1273 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1274 MLX5_MODIFICATION_TYPE_SET, error);
1278 * Convert modify-header set IPv4 DSCP action to DV specification.
1280 * @param[in,out] resource
1281 * Pointer to the modify-header resource.
1283 * Pointer to action specification.
1285 * Pointer to the error structure.
1288 * 0 on success, a negative errno value otherwise and rte_errno is set.
1291 flow_dv_convert_action_modify_ipv4_dscp
1292 (struct mlx5_flow_dv_modify_hdr_resource *resource,
1293 const struct rte_flow_action *action,
1294 struct rte_flow_error *error)
1296 const struct rte_flow_action_set_dscp *conf =
1297 (const struct rte_flow_action_set_dscp *)(action->conf);
1298 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1299 struct rte_flow_item_ipv4 ipv4;
1300 struct rte_flow_item_ipv4 ipv4_mask;
1302 memset(&ipv4, 0, sizeof(ipv4));
1303 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1304 ipv4.hdr.type_of_service = conf->dscp;
1305 ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1307 item.mask = &ipv4_mask;
1308 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1309 MLX5_MODIFICATION_TYPE_SET, error);
1313 * Convert modify-header set IPv6 DSCP action to DV specification.
1315 * @param[in,out] resource
1316 * Pointer to the modify-header resource.
1318 * Pointer to action specification.
1320 * Pointer to the error structure.
1323 * 0 on success, a negative errno value otherwise and rte_errno is set.
1326 flow_dv_convert_action_modify_ipv6_dscp
1327 (struct mlx5_flow_dv_modify_hdr_resource *resource,
1328 const struct rte_flow_action *action,
1329 struct rte_flow_error *error)
1331 const struct rte_flow_action_set_dscp *conf =
1332 (const struct rte_flow_action_set_dscp *)(action->conf);
1333 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1334 struct rte_flow_item_ipv6 ipv6;
1335 struct rte_flow_item_ipv6 ipv6_mask;
1337 memset(&ipv6, 0, sizeof(ipv6));
1338 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1340 * Even though the DSCP bits offset of IPv6 is not byte aligned,
1341 * rdma-core only accept the DSCP bits byte aligned start from
1342 * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1343 * bits in IPv6 case as rdma-core requires byte aligned value.
1345 ipv6.hdr.vtc_flow = conf->dscp;
1346 ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1348 item.mask = &ipv6_mask;
1349 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1350 MLX5_MODIFICATION_TYPE_SET, error);
1354 * Validate MARK item.
1357 * Pointer to the rte_eth_dev structure.
1359 * Item specification.
1361 * Attributes of flow that includes this item.
1363 * Pointer to error structure.
1366 * 0 on success, a negative errno value otherwise and rte_errno is set.
1369 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1370 const struct rte_flow_item *item,
1371 const struct rte_flow_attr *attr __rte_unused,
1372 struct rte_flow_error *error)
1374 struct mlx5_priv *priv = dev->data->dev_private;
1375 struct mlx5_dev_config *config = &priv->config;
1376 const struct rte_flow_item_mark *spec = item->spec;
1377 const struct rte_flow_item_mark *mask = item->mask;
1378 const struct rte_flow_item_mark nic_mask = {
1379 .id = priv->sh->dv_mark_mask,
1383 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1384 return rte_flow_error_set(error, ENOTSUP,
1385 RTE_FLOW_ERROR_TYPE_ITEM, item,
1386 "extended metadata feature"
1388 if (!mlx5_flow_ext_mreg_supported(dev))
1389 return rte_flow_error_set(error, ENOTSUP,
1390 RTE_FLOW_ERROR_TYPE_ITEM, item,
1391 "extended metadata register"
1392 " isn't supported");
1394 return rte_flow_error_set(error, ENOTSUP,
1395 RTE_FLOW_ERROR_TYPE_ITEM, item,
1396 "extended metadata register"
1397 " isn't available");
1398 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1402 return rte_flow_error_set(error, EINVAL,
1403 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1405 "data cannot be empty");
1406 if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1407 return rte_flow_error_set(error, EINVAL,
1408 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1410 "mark id exceeds the limit");
1414 return rte_flow_error_set(error, EINVAL,
1415 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1416 "mask cannot be zero");
1418 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1419 (const uint8_t *)&nic_mask,
1420 sizeof(struct rte_flow_item_mark),
1428 * Validate META item.
1431 * Pointer to the rte_eth_dev structure.
1433 * Item specification.
1435 * Attributes of flow that includes this item.
1437 * Pointer to error structure.
1440 * 0 on success, a negative errno value otherwise and rte_errno is set.
1443 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1444 const struct rte_flow_item *item,
1445 const struct rte_flow_attr *attr,
1446 struct rte_flow_error *error)
1448 struct mlx5_priv *priv = dev->data->dev_private;
1449 struct mlx5_dev_config *config = &priv->config;
1450 const struct rte_flow_item_meta *spec = item->spec;
1451 const struct rte_flow_item_meta *mask = item->mask;
1452 struct rte_flow_item_meta nic_mask = {
1459 return rte_flow_error_set(error, EINVAL,
1460 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1462 "data cannot be empty");
1463 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1464 if (!mlx5_flow_ext_mreg_supported(dev))
1465 return rte_flow_error_set(error, ENOTSUP,
1466 RTE_FLOW_ERROR_TYPE_ITEM, item,
1467 "extended metadata register"
1468 " isn't supported");
1469 reg = flow_dv_get_metadata_reg(dev, attr, error);
1473 return rte_flow_error_set(error, ENOTSUP,
1474 RTE_FLOW_ERROR_TYPE_ITEM, item,
1478 nic_mask.data = priv->sh->dv_meta_mask;
1479 } else if (attr->transfer) {
1480 return rte_flow_error_set(error, ENOTSUP,
1481 RTE_FLOW_ERROR_TYPE_ITEM, item,
1482 "extended metadata feature "
1483 "should be enabled when "
1484 "meta item is requested "
1485 "with e-switch mode ");
1488 mask = &rte_flow_item_meta_mask;
1490 return rte_flow_error_set(error, EINVAL,
1491 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1492 "mask cannot be zero");
1494 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1495 (const uint8_t *)&nic_mask,
1496 sizeof(struct rte_flow_item_meta),
1502 * Validate TAG item.
1505 * Pointer to the rte_eth_dev structure.
1507 * Item specification.
1509 * Attributes of flow that includes this item.
1511 * Pointer to error structure.
1514 * 0 on success, a negative errno value otherwise and rte_errno is set.
1517 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1518 const struct rte_flow_item *item,
1519 const struct rte_flow_attr *attr __rte_unused,
1520 struct rte_flow_error *error)
1522 const struct rte_flow_item_tag *spec = item->spec;
1523 const struct rte_flow_item_tag *mask = item->mask;
1524 const struct rte_flow_item_tag nic_mask = {
1525 .data = RTE_BE32(UINT32_MAX),
1530 if (!mlx5_flow_ext_mreg_supported(dev))
1531 return rte_flow_error_set(error, ENOTSUP,
1532 RTE_FLOW_ERROR_TYPE_ITEM, item,
1533 "extensive metadata register"
1534 " isn't supported");
1536 return rte_flow_error_set(error, EINVAL,
1537 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1539 "data cannot be empty");
1541 mask = &rte_flow_item_tag_mask;
1543 return rte_flow_error_set(error, EINVAL,
1544 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1545 "mask cannot be zero");
1547 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1548 (const uint8_t *)&nic_mask,
1549 sizeof(struct rte_flow_item_tag),
1553 if (mask->index != 0xff)
1554 return rte_flow_error_set(error, EINVAL,
1555 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1556 "partial mask for tag index"
1557 " is not supported");
1558 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1561 MLX5_ASSERT(ret != REG_NON);
1566 * Validate vport item.
1569 * Pointer to the rte_eth_dev structure.
1571 * Item specification.
1573 * Attributes of flow that includes this item.
1574 * @param[in] item_flags
1575 * Bit-fields that holds the items detected until now.
1577 * Pointer to error structure.
1580 * 0 on success, a negative errno value otherwise and rte_errno is set.
1583 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1584 const struct rte_flow_item *item,
1585 const struct rte_flow_attr *attr,
1586 uint64_t item_flags,
1587 struct rte_flow_error *error)
1589 const struct rte_flow_item_port_id *spec = item->spec;
1590 const struct rte_flow_item_port_id *mask = item->mask;
1591 const struct rte_flow_item_port_id switch_mask = {
1594 struct mlx5_priv *esw_priv;
1595 struct mlx5_priv *dev_priv;
1598 if (!attr->transfer)
1599 return rte_flow_error_set(error, EINVAL,
1600 RTE_FLOW_ERROR_TYPE_ITEM,
1602 "match on port id is valid only"
1603 " when transfer flag is enabled");
1604 if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1605 return rte_flow_error_set(error, ENOTSUP,
1606 RTE_FLOW_ERROR_TYPE_ITEM, item,
1607 "multiple source ports are not"
1610 mask = &switch_mask;
1611 if (mask->id != 0xffffffff)
1612 return rte_flow_error_set(error, ENOTSUP,
1613 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1615 "no support for partial mask on"
1617 ret = mlx5_flow_item_acceptable
1618 (item, (const uint8_t *)mask,
1619 (const uint8_t *)&rte_flow_item_port_id_mask,
1620 sizeof(struct rte_flow_item_port_id),
1626 esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1628 return rte_flow_error_set(error, rte_errno,
1629 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1630 "failed to obtain E-Switch info for"
1632 dev_priv = mlx5_dev_to_eswitch_info(dev);
1634 return rte_flow_error_set(error, rte_errno,
1635 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1637 "failed to obtain E-Switch info");
1638 if (esw_priv->domain_id != dev_priv->domain_id)
1639 return rte_flow_error_set(error, EINVAL,
1640 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1641 "cannot match on a port from a"
1642 " different E-Switch");
1647 * Validate VLAN item.
1650 * Item specification.
1651 * @param[in] item_flags
1652 * Bit-fields that holds the items detected until now.
1654 * Ethernet device flow is being created on.
1656 * Pointer to error structure.
1659 * 0 on success, a negative errno value otherwise and rte_errno is set.
1662 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1663 uint64_t item_flags,
1664 struct rte_eth_dev *dev,
1665 struct rte_flow_error *error)
1667 const struct rte_flow_item_vlan *mask = item->mask;
1668 const struct rte_flow_item_vlan nic_mask = {
1669 .tci = RTE_BE16(UINT16_MAX),
1670 .inner_type = RTE_BE16(UINT16_MAX),
1672 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1674 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1675 MLX5_FLOW_LAYER_INNER_L4) :
1676 (MLX5_FLOW_LAYER_OUTER_L3 |
1677 MLX5_FLOW_LAYER_OUTER_L4);
1678 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1679 MLX5_FLOW_LAYER_OUTER_VLAN;
1681 if (item_flags & vlanm)
1682 return rte_flow_error_set(error, EINVAL,
1683 RTE_FLOW_ERROR_TYPE_ITEM, item,
1684 "multiple VLAN layers not supported");
1685 else if ((item_flags & l34m) != 0)
1686 return rte_flow_error_set(error, EINVAL,
1687 RTE_FLOW_ERROR_TYPE_ITEM, item,
1688 "VLAN cannot follow L3/L4 layer");
1690 mask = &rte_flow_item_vlan_mask;
1691 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1692 (const uint8_t *)&nic_mask,
1693 sizeof(struct rte_flow_item_vlan),
1697 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1698 struct mlx5_priv *priv = dev->data->dev_private;
1700 if (priv->vmwa_context) {
1702 * Non-NULL context means we have a virtual machine
1703 * and SR-IOV enabled, we have to create VLAN interface
1704 * to make hypervisor to setup E-Switch vport
1705 * context correctly. We avoid creating the multiple
1706 * VLAN interfaces, so we cannot support VLAN tag mask.
1708 return rte_flow_error_set(error, EINVAL,
1709 RTE_FLOW_ERROR_TYPE_ITEM,
1711 "VLAN tag mask is not"
1712 " supported in virtual"
1720 * GTP flags are contained in 1 byte of the format:
1721 * -------------------------------------------
1722 * | bit | 0 - 2 | 3 | 4 | 5 | 6 | 7 |
1723 * |-----------------------------------------|
1724 * | value | Version | PT | Res | E | S | PN |
1725 * -------------------------------------------
1727 * Matching is supported only for GTP flags E, S, PN.
1729 #define MLX5_GTP_FLAGS_MASK 0x07
1732 * Validate GTP item.
1735 * Pointer to the rte_eth_dev structure.
1737 * Item specification.
1738 * @param[in] item_flags
1739 * Bit-fields that holds the items detected until now.
1741 * Pointer to error structure.
1744 * 0 on success, a negative errno value otherwise and rte_errno is set.
1747 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1748 const struct rte_flow_item *item,
1749 uint64_t item_flags,
1750 struct rte_flow_error *error)
1752 struct mlx5_priv *priv = dev->data->dev_private;
1753 const struct rte_flow_item_gtp *spec = item->spec;
1754 const struct rte_flow_item_gtp *mask = item->mask;
1755 const struct rte_flow_item_gtp nic_mask = {
1756 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1758 .teid = RTE_BE32(0xffffffff),
1761 if (!priv->config.hca_attr.tunnel_stateless_gtp)
1762 return rte_flow_error_set(error, ENOTSUP,
1763 RTE_FLOW_ERROR_TYPE_ITEM, item,
1764 "GTP support is not enabled");
1765 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1766 return rte_flow_error_set(error, ENOTSUP,
1767 RTE_FLOW_ERROR_TYPE_ITEM, item,
1768 "multiple tunnel layers not"
1770 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1771 return rte_flow_error_set(error, EINVAL,
1772 RTE_FLOW_ERROR_TYPE_ITEM, item,
1773 "no outer UDP layer found");
1775 mask = &rte_flow_item_gtp_mask;
1776 if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1777 return rte_flow_error_set(error, ENOTSUP,
1778 RTE_FLOW_ERROR_TYPE_ITEM, item,
1779 "Match is supported for GTP"
1781 return mlx5_flow_item_acceptable
1782 (item, (const uint8_t *)mask,
1783 (const uint8_t *)&nic_mask,
1784 sizeof(struct rte_flow_item_gtp),
1789 * Validate the pop VLAN action.
1792 * Pointer to the rte_eth_dev structure.
1793 * @param[in] action_flags
1794 * Holds the actions detected until now.
1796 * Pointer to the pop vlan action.
1797 * @param[in] item_flags
1798 * The items found in this flow rule.
1800 * Pointer to flow attributes.
1802 * Pointer to error structure.
1805 * 0 on success, a negative errno value otherwise and rte_errno is set.
1808 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1809 uint64_t action_flags,
1810 const struct rte_flow_action *action,
1811 uint64_t item_flags,
1812 const struct rte_flow_attr *attr,
1813 struct rte_flow_error *error)
1815 const struct mlx5_priv *priv = dev->data->dev_private;
1819 if (!priv->sh->pop_vlan_action)
1820 return rte_flow_error_set(error, ENOTSUP,
1821 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1823 "pop vlan action is not supported");
1825 return rte_flow_error_set(error, ENOTSUP,
1826 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1828 "pop vlan action not supported for "
1830 if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1831 return rte_flow_error_set(error, ENOTSUP,
1832 RTE_FLOW_ERROR_TYPE_ACTION, action,
1833 "no support for multiple VLAN "
1835 /* Pop VLAN with preceding Decap requires inner header with VLAN. */
1836 if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
1837 !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
1838 return rte_flow_error_set(error, ENOTSUP,
1839 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1841 "cannot pop vlan after decap without "
1842 "match on inner vlan in the flow");
1843 /* Pop VLAN without preceding Decap requires outer header with VLAN. */
1844 if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
1845 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1846 return rte_flow_error_set(error, ENOTSUP,
1847 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1849 "cannot pop vlan without a "
1850 "match on (outer) vlan in the flow");
1851 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1852 return rte_flow_error_set(error, EINVAL,
1853 RTE_FLOW_ERROR_TYPE_ACTION, action,
1854 "wrong action order, port_id should "
1855 "be after pop VLAN action");
1856 if (!attr->transfer && priv->representor)
1857 return rte_flow_error_set(error, ENOTSUP,
1858 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1859 "pop vlan action for VF representor "
1860 "not supported on NIC table");
1865 * Get VLAN default info from vlan match info.
1868 * the list of item specifications.
1870 * pointer VLAN info to fill to.
1873 * 0 on success, a negative errno value otherwise and rte_errno is set.
1876 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1877 struct rte_vlan_hdr *vlan)
1879 const struct rte_flow_item_vlan nic_mask = {
1880 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1881 MLX5DV_FLOW_VLAN_VID_MASK),
1882 .inner_type = RTE_BE16(0xffff),
1887 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1888 int type = items->type;
1890 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1891 type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1894 if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1895 const struct rte_flow_item_vlan *vlan_m = items->mask;
1896 const struct rte_flow_item_vlan *vlan_v = items->spec;
1898 /* If VLAN item in pattern doesn't contain data, return here. */
1903 /* Only full match values are accepted */
1904 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1905 MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1906 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1908 rte_be_to_cpu_16(vlan_v->tci &
1909 MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1911 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1912 MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1913 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1915 rte_be_to_cpu_16(vlan_v->tci &
1916 MLX5DV_FLOW_VLAN_VID_MASK_BE);
1918 if (vlan_m->inner_type == nic_mask.inner_type)
1919 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1920 vlan_m->inner_type);
1925 * Validate the push VLAN action.
1928 * Pointer to the rte_eth_dev structure.
1929 * @param[in] action_flags
1930 * Holds the actions detected until now.
1931 * @param[in] item_flags
1932 * The items found in this flow rule.
1934 * Pointer to the action structure.
1936 * Pointer to flow attributes
1938 * Pointer to error structure.
1941 * 0 on success, a negative errno value otherwise and rte_errno is set.
1944 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1945 uint64_t action_flags,
1946 const struct rte_flow_item_vlan *vlan_m,
1947 const struct rte_flow_action *action,
1948 const struct rte_flow_attr *attr,
1949 struct rte_flow_error *error)
1951 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1952 const struct mlx5_priv *priv = dev->data->dev_private;
1954 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1955 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1956 return rte_flow_error_set(error, EINVAL,
1957 RTE_FLOW_ERROR_TYPE_ACTION, action,
1958 "invalid vlan ethertype");
1959 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1960 return rte_flow_error_set(error, EINVAL,
1961 RTE_FLOW_ERROR_TYPE_ACTION, action,
1962 "wrong action order, port_id should "
1963 "be after push VLAN");
1964 if (!attr->transfer && priv->representor)
1965 return rte_flow_error_set(error, ENOTSUP,
1966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1967 "push vlan action for VF representor "
1968 "not supported on NIC table");
1970 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1971 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1972 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1973 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1974 !(mlx5_flow_find_action
1975 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1976 return rte_flow_error_set(error, EINVAL,
1977 RTE_FLOW_ERROR_TYPE_ACTION, action,
1978 "not full match mask on VLAN PCP and "
1979 "there is no of_set_vlan_pcp action, "
1980 "push VLAN action cannot figure out "
1983 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1984 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1985 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1986 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1987 !(mlx5_flow_find_action
1988 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1989 return rte_flow_error_set(error, EINVAL,
1990 RTE_FLOW_ERROR_TYPE_ACTION, action,
1991 "not full match mask on VLAN VID and "
1992 "there is no of_set_vlan_vid action, "
1993 "push VLAN action cannot figure out "
2000 * Validate the set VLAN PCP.
2002 * @param[in] action_flags
2003 * Holds the actions detected until now.
2004 * @param[in] actions
2005 * Pointer to the list of actions remaining in the flow rule.
2007 * Pointer to error structure.
2010 * 0 on success, a negative errno value otherwise and rte_errno is set.
2013 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2014 const struct rte_flow_action actions[],
2015 struct rte_flow_error *error)
2017 const struct rte_flow_action *action = actions;
2018 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2020 if (conf->vlan_pcp > 7)
2021 return rte_flow_error_set(error, EINVAL,
2022 RTE_FLOW_ERROR_TYPE_ACTION, action,
2023 "VLAN PCP value is too big");
2024 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2025 return rte_flow_error_set(error, ENOTSUP,
2026 RTE_FLOW_ERROR_TYPE_ACTION, action,
2027 "set VLAN PCP action must follow "
2028 "the push VLAN action");
2029 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2030 return rte_flow_error_set(error, ENOTSUP,
2031 RTE_FLOW_ERROR_TYPE_ACTION, action,
2032 "Multiple VLAN PCP modification are "
2034 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2035 return rte_flow_error_set(error, EINVAL,
2036 RTE_FLOW_ERROR_TYPE_ACTION, action,
2037 "wrong action order, port_id should "
2038 "be after set VLAN PCP");
2043 * Validate the set VLAN VID.
2045 * @param[in] item_flags
2046 * Holds the items detected in this rule.
2047 * @param[in] action_flags
2048 * Holds the actions detected until now.
2049 * @param[in] actions
2050 * Pointer to the list of actions remaining in the flow rule.
2052 * Pointer to error structure.
2055 * 0 on success, a negative errno value otherwise and rte_errno is set.
2058 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2059 uint64_t action_flags,
2060 const struct rte_flow_action actions[],
2061 struct rte_flow_error *error)
2063 const struct rte_flow_action *action = actions;
2064 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2066 if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2067 return rte_flow_error_set(error, EINVAL,
2068 RTE_FLOW_ERROR_TYPE_ACTION, action,
2069 "VLAN VID value is too big");
2070 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2071 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2072 return rte_flow_error_set(error, ENOTSUP,
2073 RTE_FLOW_ERROR_TYPE_ACTION, action,
2074 "set VLAN VID action must follow push"
2075 " VLAN action or match on VLAN item");
2076 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2077 return rte_flow_error_set(error, ENOTSUP,
2078 RTE_FLOW_ERROR_TYPE_ACTION, action,
2079 "Multiple VLAN VID modifications are "
2081 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2082 return rte_flow_error_set(error, EINVAL,
2083 RTE_FLOW_ERROR_TYPE_ACTION, action,
2084 "wrong action order, port_id should "
2085 "be after set VLAN VID");
2090 * Validate the FLAG action.
2093 * Pointer to the rte_eth_dev structure.
2094 * @param[in] action_flags
2095 * Holds the actions detected until now.
2097 * Pointer to flow attributes
2099 * Pointer to error structure.
2102 * 0 on success, a negative errno value otherwise and rte_errno is set.
2105 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2106 uint64_t action_flags,
2107 const struct rte_flow_attr *attr,
2108 struct rte_flow_error *error)
2110 struct mlx5_priv *priv = dev->data->dev_private;
2111 struct mlx5_dev_config *config = &priv->config;
2114 /* Fall back if no extended metadata register support. */
2115 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2116 return mlx5_flow_validate_action_flag(action_flags, attr,
2118 /* Extensive metadata mode requires registers. */
2119 if (!mlx5_flow_ext_mreg_supported(dev))
2120 return rte_flow_error_set(error, ENOTSUP,
2121 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2122 "no metadata registers "
2123 "to support flag action");
2124 if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2125 return rte_flow_error_set(error, ENOTSUP,
2126 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2127 "extended metadata register"
2128 " isn't available");
2129 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2132 MLX5_ASSERT(ret > 0);
2133 if (action_flags & MLX5_FLOW_ACTION_MARK)
2134 return rte_flow_error_set(error, EINVAL,
2135 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2136 "can't mark and flag in same flow");
2137 if (action_flags & MLX5_FLOW_ACTION_FLAG)
2138 return rte_flow_error_set(error, EINVAL,
2139 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2141 " actions in same flow");
2146 * Validate MARK action.
2149 * Pointer to the rte_eth_dev structure.
2151 * Pointer to action.
2152 * @param[in] action_flags
2153 * Holds the actions detected until now.
2155 * Pointer to flow attributes
2157 * Pointer to error structure.
2160 * 0 on success, a negative errno value otherwise and rte_errno is set.
2163 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2164 const struct rte_flow_action *action,
2165 uint64_t action_flags,
2166 const struct rte_flow_attr *attr,
2167 struct rte_flow_error *error)
2169 struct mlx5_priv *priv = dev->data->dev_private;
2170 struct mlx5_dev_config *config = &priv->config;
2171 const struct rte_flow_action_mark *mark = action->conf;
2174 /* Fall back if no extended metadata register support. */
2175 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2176 return mlx5_flow_validate_action_mark(action, action_flags,
2178 /* Extensive metadata mode requires registers. */
2179 if (!mlx5_flow_ext_mreg_supported(dev))
2180 return rte_flow_error_set(error, ENOTSUP,
2181 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2182 "no metadata registers "
2183 "to support mark action");
2184 if (!priv->sh->dv_mark_mask)
2185 return rte_flow_error_set(error, ENOTSUP,
2186 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2187 "extended metadata register"
2188 " isn't available");
2189 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2192 MLX5_ASSERT(ret > 0);
2194 return rte_flow_error_set(error, EINVAL,
2195 RTE_FLOW_ERROR_TYPE_ACTION, action,
2196 "configuration cannot be null");
2197 if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2198 return rte_flow_error_set(error, EINVAL,
2199 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2201 "mark id exceeds the limit");
2202 if (action_flags & MLX5_FLOW_ACTION_FLAG)
2203 return rte_flow_error_set(error, EINVAL,
2204 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2205 "can't flag and mark in same flow");
2206 if (action_flags & MLX5_FLOW_ACTION_MARK)
2207 return rte_flow_error_set(error, EINVAL,
2208 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2209 "can't have 2 mark actions in same"
2215 * Validate SET_META action.
2218 * Pointer to the rte_eth_dev structure.
2220 * Pointer to the action structure.
2221 * @param[in] action_flags
2222 * Holds the actions detected until now.
2224 * Pointer to flow attributes
2226 * Pointer to error structure.
2229 * 0 on success, a negative errno value otherwise and rte_errno is set.
2232 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2233 const struct rte_flow_action *action,
2234 uint64_t action_flags __rte_unused,
2235 const struct rte_flow_attr *attr,
2236 struct rte_flow_error *error)
2238 const struct rte_flow_action_set_meta *conf;
2239 uint32_t nic_mask = UINT32_MAX;
2242 if (!mlx5_flow_ext_mreg_supported(dev))
2243 return rte_flow_error_set(error, ENOTSUP,
2244 RTE_FLOW_ERROR_TYPE_ACTION, action,
2245 "extended metadata register"
2246 " isn't supported");
2247 reg = flow_dv_get_metadata_reg(dev, attr, error);
2250 if (reg != REG_A && reg != REG_B) {
2251 struct mlx5_priv *priv = dev->data->dev_private;
2253 nic_mask = priv->sh->dv_meta_mask;
2255 if (!(action->conf))
2256 return rte_flow_error_set(error, EINVAL,
2257 RTE_FLOW_ERROR_TYPE_ACTION, action,
2258 "configuration cannot be null");
2259 conf = (const struct rte_flow_action_set_meta *)action->conf;
2261 return rte_flow_error_set(error, EINVAL,
2262 RTE_FLOW_ERROR_TYPE_ACTION, action,
2263 "zero mask doesn't have any effect");
2264 if (conf->mask & ~nic_mask)
2265 return rte_flow_error_set(error, EINVAL,
2266 RTE_FLOW_ERROR_TYPE_ACTION, action,
2267 "meta data must be within reg C0");
2272 * Validate SET_TAG action.
2275 * Pointer to the rte_eth_dev structure.
2277 * Pointer to the action structure.
2278 * @param[in] action_flags
2279 * Holds the actions detected until now.
2281 * Pointer to flow attributes
2283 * Pointer to error structure.
2286 * 0 on success, a negative errno value otherwise and rte_errno is set.
2289 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2290 const struct rte_flow_action *action,
2291 uint64_t action_flags,
2292 const struct rte_flow_attr *attr,
2293 struct rte_flow_error *error)
2295 const struct rte_flow_action_set_tag *conf;
2296 const uint64_t terminal_action_flags =
2297 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2298 MLX5_FLOW_ACTION_RSS;
2301 if (!mlx5_flow_ext_mreg_supported(dev))
2302 return rte_flow_error_set(error, ENOTSUP,
2303 RTE_FLOW_ERROR_TYPE_ACTION, action,
2304 "extensive metadata register"
2305 " isn't supported");
2306 if (!(action->conf))
2307 return rte_flow_error_set(error, EINVAL,
2308 RTE_FLOW_ERROR_TYPE_ACTION, action,
2309 "configuration cannot be null");
2310 conf = (const struct rte_flow_action_set_tag *)action->conf;
2312 return rte_flow_error_set(error, EINVAL,
2313 RTE_FLOW_ERROR_TYPE_ACTION, action,
2314 "zero mask doesn't have any effect");
2315 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2318 if (!attr->transfer && attr->ingress &&
2319 (action_flags & terminal_action_flags))
2320 return rte_flow_error_set(error, EINVAL,
2321 RTE_FLOW_ERROR_TYPE_ACTION, action,
2322 "set_tag has no effect"
2323 " with terminal actions");
2328 * Validate count action.
2331 * Pointer to rte_eth_dev structure.
2333 * Pointer to error structure.
2336 * 0 on success, a negative errno value otherwise and rte_errno is set.
2339 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2340 struct rte_flow_error *error)
2342 struct mlx5_priv *priv = dev->data->dev_private;
2344 if (!priv->config.devx)
2346 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2350 return rte_flow_error_set
2352 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2354 "count action not supported");
2358 * Validate the L2 encap action.
2361 * Pointer to the rte_eth_dev structure.
2362 * @param[in] action_flags
2363 * Holds the actions detected until now.
2365 * Pointer to the action structure.
2367 * Pointer to flow attributes.
2369 * Pointer to error structure.
2372 * 0 on success, a negative errno value otherwise and rte_errno is set.
2375 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2376 uint64_t action_flags,
2377 const struct rte_flow_action *action,
2378 const struct rte_flow_attr *attr,
2379 struct rte_flow_error *error)
2381 const struct mlx5_priv *priv = dev->data->dev_private;
2383 if (!(action->conf))
2384 return rte_flow_error_set(error, EINVAL,
2385 RTE_FLOW_ERROR_TYPE_ACTION, action,
2386 "configuration cannot be null");
2387 if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2388 return rte_flow_error_set(error, EINVAL,
2389 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2390 "can only have a single encap action "
2392 if (!attr->transfer && priv->representor)
2393 return rte_flow_error_set(error, ENOTSUP,
2394 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2395 "encap action for VF representor "
2396 "not supported on NIC table");
2401 * Validate a decap action.
2404 * Pointer to the rte_eth_dev structure.
2405 * @param[in] action_flags
2406 * Holds the actions detected until now.
2408 * Pointer to flow attributes
2410 * Pointer to error structure.
2413 * 0 on success, a negative errno value otherwise and rte_errno is set.
2416 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2417 uint64_t action_flags,
2418 const struct rte_flow_attr *attr,
2419 struct rte_flow_error *error)
2421 const struct mlx5_priv *priv = dev->data->dev_private;
2423 if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2424 !priv->config.decap_en)
2425 return rte_flow_error_set(error, ENOTSUP,
2426 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2427 "decap is not enabled");
2428 if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2429 return rte_flow_error_set(error, ENOTSUP,
2430 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2432 MLX5_FLOW_ACTION_DECAP ? "can only "
2433 "have a single decap action" : "decap "
2434 "after encap is not supported");
2435 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2436 return rte_flow_error_set(error, EINVAL,
2437 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2438 "can't have decap action after"
2441 return rte_flow_error_set(error, ENOTSUP,
2442 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2444 "decap action not supported for "
2446 if (!attr->transfer && priv->representor)
2447 return rte_flow_error_set(error, ENOTSUP,
2448 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2449 "decap action for VF representor "
2450 "not supported on NIC table");
2454 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2457 * Validate the raw encap and decap actions.
2460 * Pointer to the rte_eth_dev structure.
2462 * Pointer to the decap action.
2464 * Pointer to the encap action.
2466 * Pointer to flow attributes
2467 * @param[in/out] action_flags
2468 * Holds the actions detected until now.
2469 * @param[out] actions_n
2470 * pointer to the number of actions counter.
2472 * Pointer to error structure.
2475 * 0 on success, a negative errno value otherwise and rte_errno is set.
2478 flow_dv_validate_action_raw_encap_decap
2479 (struct rte_eth_dev *dev,
2480 const struct rte_flow_action_raw_decap *decap,
2481 const struct rte_flow_action_raw_encap *encap,
2482 const struct rte_flow_attr *attr, uint64_t *action_flags,
2483 int *actions_n, struct rte_flow_error *error)
2485 const struct mlx5_priv *priv = dev->data->dev_private;
2488 if (encap && (!encap->size || !encap->data))
2489 return rte_flow_error_set(error, EINVAL,
2490 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2491 "raw encap data cannot be empty");
2492 if (decap && encap) {
2493 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2494 encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2497 else if (encap->size <=
2498 MLX5_ENCAPSULATION_DECISION_SIZE &&
2500 MLX5_ENCAPSULATION_DECISION_SIZE)
2503 else if (encap->size >
2504 MLX5_ENCAPSULATION_DECISION_SIZE &&
2506 MLX5_ENCAPSULATION_DECISION_SIZE)
2507 /* 2 L2 actions: encap and decap. */
2510 return rte_flow_error_set(error,
2512 RTE_FLOW_ERROR_TYPE_ACTION,
2513 NULL, "unsupported too small "
2514 "raw decap and too small raw "
2515 "encap combination");
2518 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2522 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2526 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2527 return rte_flow_error_set(error, ENOTSUP,
2528 RTE_FLOW_ERROR_TYPE_ACTION,
2530 "small raw encap size");
2531 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2532 return rte_flow_error_set(error, EINVAL,
2533 RTE_FLOW_ERROR_TYPE_ACTION,
2535 "more than one encap action");
2536 if (!attr->transfer && priv->representor)
2537 return rte_flow_error_set
2539 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2540 "encap action for VF representor "
2541 "not supported on NIC table");
2542 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2549 * Match encap_decap resource.
2552 * Pointer to exist resource entry object.
2554 * Pointer to new encap_decap resource.
2557 * 0 on matching, -1 otherwise.
2560 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2562 struct mlx5_flow_dv_encap_decap_resource *resource;
2563 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2565 resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2566 cache_resource = container_of(entry,
2567 struct mlx5_flow_dv_encap_decap_resource,
2569 if (resource->entry.key == cache_resource->entry.key &&
2570 resource->reformat_type == cache_resource->reformat_type &&
2571 resource->ft_type == cache_resource->ft_type &&
2572 resource->flags == cache_resource->flags &&
2573 resource->size == cache_resource->size &&
2574 !memcmp((const void *)resource->buf,
2575 (const void *)cache_resource->buf,
2582 * Find existing encap/decap resource or create and register a new one.
2584 * @param[in, out] dev
2585 * Pointer to rte_eth_dev structure.
2586 * @param[in, out] resource
2587 * Pointer to encap/decap resource.
2588 * @parm[in, out] dev_flow
2589 * Pointer to the dev_flow.
2591 * pointer to error structure.
2594 * 0 on success otherwise -errno and errno is set.
2597 flow_dv_encap_decap_resource_register
2598 (struct rte_eth_dev *dev,
2599 struct mlx5_flow_dv_encap_decap_resource *resource,
2600 struct mlx5_flow *dev_flow,
2601 struct rte_flow_error *error)
2603 struct mlx5_priv *priv = dev->data->dev_private;
2604 struct mlx5_dev_ctx_shared *sh = priv->sh;
2605 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2606 struct mlx5dv_dr_domain *domain;
2607 struct mlx5_hlist_entry *entry;
2608 union mlx5_flow_encap_decap_key encap_decap_key = {
2610 .ft_type = resource->ft_type,
2611 .refmt_type = resource->reformat_type,
2612 .buf_size = resource->size,
2613 .table_level = !!dev_flow->dv.group,
2619 resource->flags = dev_flow->dv.group ? 0 : 1;
2620 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2621 domain = sh->fdb_domain;
2622 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2623 domain = sh->rx_domain;
2625 domain = sh->tx_domain;
2626 encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2628 resource->entry.key = encap_decap_key.v64;
2629 /* Lookup a matching resource from cache. */
2630 entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2631 flow_dv_encap_decap_resource_match,
2634 cache_resource = container_of(entry,
2635 struct mlx5_flow_dv_encap_decap_resource, entry);
2636 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2637 (void *)cache_resource,
2638 rte_atomic32_read(&cache_resource->refcnt));
2639 rte_atomic32_inc(&cache_resource->refcnt);
2640 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2641 dev_flow->dv.encap_decap = cache_resource;
2644 /* Register new encap/decap resource. */
2645 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2646 &dev_flow->handle->dvh.rix_encap_decap);
2647 if (!cache_resource)
2648 return rte_flow_error_set(error, ENOMEM,
2649 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2650 "cannot allocate resource memory");
2651 *cache_resource = *resource;
2652 cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2653 ret = mlx5_flow_os_create_flow_action_packet_reformat
2654 (sh->ctx, domain, cache_resource,
2655 &cache_resource->action);
2657 mlx5_free(cache_resource);
2658 return rte_flow_error_set(error, ENOMEM,
2659 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2660 NULL, "cannot create action");
2662 rte_atomic32_init(&cache_resource->refcnt);
2663 rte_atomic32_inc(&cache_resource->refcnt);
2664 if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2665 flow_dv_encap_decap_resource_match,
2666 (void *)cache_resource)) {
2667 claim_zero(mlx5_flow_os_destroy_flow_action
2668 (cache_resource->action));
2669 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2670 cache_resource->idx);
2671 return rte_flow_error_set(error, EEXIST,
2672 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2673 NULL, "action exist");
2675 dev_flow->dv.encap_decap = cache_resource;
2676 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2677 (void *)cache_resource,
2678 rte_atomic32_read(&cache_resource->refcnt));
2683 * Find existing table jump resource or create and register a new one.
2685 * @param[in, out] dev
2686 * Pointer to rte_eth_dev structure.
2687 * @param[in, out] tbl
2688 * Pointer to flow table resource.
2689 * @parm[in, out] dev_flow
2690 * Pointer to the dev_flow.
2692 * pointer to error structure.
2695 * 0 on success otherwise -errno and errno is set.
2698 flow_dv_jump_tbl_resource_register
2699 (struct rte_eth_dev *dev __rte_unused,
2700 struct mlx5_flow_tbl_resource *tbl,
2701 struct mlx5_flow *dev_flow,
2702 struct rte_flow_error *error)
2704 struct mlx5_flow_tbl_data_entry *tbl_data =
2705 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2709 cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2711 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2712 (tbl->obj, &tbl_data->jump.action);
2714 return rte_flow_error_set(error, ENOMEM,
2715 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2716 NULL, "cannot create jump action");
2717 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2718 (void *)&tbl_data->jump, cnt);
2720 /* old jump should not make the table ref++. */
2721 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2722 MLX5_ASSERT(tbl_data->jump.action);
2723 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2724 (void *)&tbl_data->jump, cnt);
2726 rte_atomic32_inc(&tbl_data->jump.refcnt);
2727 dev_flow->handle->rix_jump = tbl_data->idx;
2728 dev_flow->dv.jump = &tbl_data->jump;
2733 * Find existing default miss resource or create and register a new one.
2735 * @param[in, out] dev
2736 * Pointer to rte_eth_dev structure.
2738 * pointer to error structure.
2741 * 0 on success otherwise -errno and errno is set.
2744 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2745 struct rte_flow_error *error)
2747 struct mlx5_priv *priv = dev->data->dev_private;
2748 struct mlx5_dev_ctx_shared *sh = priv->sh;
2749 struct mlx5_flow_default_miss_resource *cache_resource =
2751 int cnt = rte_atomic32_read(&cache_resource->refcnt);
2754 MLX5_ASSERT(cache_resource->action);
2755 cache_resource->action =
2756 mlx5_glue->dr_create_flow_action_default_miss();
2757 if (!cache_resource->action)
2758 return rte_flow_error_set(error, ENOMEM,
2759 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2760 "cannot create default miss action");
2761 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2762 (void *)cache_resource->action, cnt);
2764 rte_atomic32_inc(&cache_resource->refcnt);
2769 * Find existing table port ID resource or create and register a new one.
2771 * @param[in, out] dev
2772 * Pointer to rte_eth_dev structure.
2773 * @param[in, out] resource
2774 * Pointer to port ID action resource.
2775 * @parm[in, out] dev_flow
2776 * Pointer to the dev_flow.
2778 * pointer to error structure.
2781 * 0 on success otherwise -errno and errno is set.
2784 flow_dv_port_id_action_resource_register
2785 (struct rte_eth_dev *dev,
2786 struct mlx5_flow_dv_port_id_action_resource *resource,
2787 struct mlx5_flow *dev_flow,
2788 struct rte_flow_error *error)
2790 struct mlx5_priv *priv = dev->data->dev_private;
2791 struct mlx5_dev_ctx_shared *sh = priv->sh;
2792 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2796 /* Lookup a matching resource from cache. */
2797 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2798 idx, cache_resource, next) {
2799 if (resource->port_id == cache_resource->port_id) {
2800 DRV_LOG(DEBUG, "port id action resource resource %p: "
2802 (void *)cache_resource,
2803 rte_atomic32_read(&cache_resource->refcnt));
2804 rte_atomic32_inc(&cache_resource->refcnt);
2805 dev_flow->handle->rix_port_id_action = idx;
2806 dev_flow->dv.port_id_action = cache_resource;
2810 /* Register new port id action resource. */
2811 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2812 &dev_flow->handle->rix_port_id_action);
2813 if (!cache_resource)
2814 return rte_flow_error_set(error, ENOMEM,
2815 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2816 "cannot allocate resource memory");
2817 *cache_resource = *resource;
2818 ret = mlx5_flow_os_create_flow_action_dest_port
2819 (priv->sh->fdb_domain, resource->port_id,
2820 &cache_resource->action);
2822 mlx5_free(cache_resource);
2823 return rte_flow_error_set(error, ENOMEM,
2824 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2825 NULL, "cannot create action");
2827 rte_atomic32_init(&cache_resource->refcnt);
2828 rte_atomic32_inc(&cache_resource->refcnt);
2829 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2830 dev_flow->handle->rix_port_id_action, cache_resource,
2832 dev_flow->dv.port_id_action = cache_resource;
2833 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2834 (void *)cache_resource,
2835 rte_atomic32_read(&cache_resource->refcnt));
2840 * Find existing push vlan resource or create and register a new one.
2842 * @param [in, out] dev
2843 * Pointer to rte_eth_dev structure.
2844 * @param[in, out] resource
2845 * Pointer to port ID action resource.
2846 * @parm[in, out] dev_flow
2847 * Pointer to the dev_flow.
2849 * pointer to error structure.
2852 * 0 on success otherwise -errno and errno is set.
2855 flow_dv_push_vlan_action_resource_register
2856 (struct rte_eth_dev *dev,
2857 struct mlx5_flow_dv_push_vlan_action_resource *resource,
2858 struct mlx5_flow *dev_flow,
2859 struct rte_flow_error *error)
2861 struct mlx5_priv *priv = dev->data->dev_private;
2862 struct mlx5_dev_ctx_shared *sh = priv->sh;
2863 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2864 struct mlx5dv_dr_domain *domain;
2868 /* Lookup a matching resource from cache. */
2869 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2870 sh->push_vlan_action_list, idx, cache_resource, next) {
2871 if (resource->vlan_tag == cache_resource->vlan_tag &&
2872 resource->ft_type == cache_resource->ft_type) {
2873 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2875 (void *)cache_resource,
2876 rte_atomic32_read(&cache_resource->refcnt));
2877 rte_atomic32_inc(&cache_resource->refcnt);
2878 dev_flow->handle->dvh.rix_push_vlan = idx;
2879 dev_flow->dv.push_vlan_res = cache_resource;
2883 /* Register new push_vlan action resource. */
2884 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2885 &dev_flow->handle->dvh.rix_push_vlan);
2886 if (!cache_resource)
2887 return rte_flow_error_set(error, ENOMEM,
2888 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2889 "cannot allocate resource memory");
2890 *cache_resource = *resource;
2891 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2892 domain = sh->fdb_domain;
2893 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2894 domain = sh->rx_domain;
2896 domain = sh->tx_domain;
2897 ret = mlx5_flow_os_create_flow_action_push_vlan
2898 (domain, resource->vlan_tag,
2899 &cache_resource->action);
2901 mlx5_free(cache_resource);
2902 return rte_flow_error_set(error, ENOMEM,
2903 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2904 NULL, "cannot create action");
2906 rte_atomic32_init(&cache_resource->refcnt);
2907 rte_atomic32_inc(&cache_resource->refcnt);
2908 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2909 &sh->push_vlan_action_list,
2910 dev_flow->handle->dvh.rix_push_vlan,
2911 cache_resource, next);
2912 dev_flow->dv.push_vlan_res = cache_resource;
2913 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2914 (void *)cache_resource,
2915 rte_atomic32_read(&cache_resource->refcnt));
2919 * Get the size of specific rte_flow_item_type hdr size
2921 * @param[in] item_type
2922 * Tested rte_flow_item_type.
2925 * sizeof struct item_type, 0 if void or irrelevant.
2928 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
2932 switch (item_type) {
2933 case RTE_FLOW_ITEM_TYPE_ETH:
2934 retval = sizeof(struct rte_ether_hdr);
2936 case RTE_FLOW_ITEM_TYPE_VLAN:
2937 retval = sizeof(struct rte_vlan_hdr);
2939 case RTE_FLOW_ITEM_TYPE_IPV4:
2940 retval = sizeof(struct rte_ipv4_hdr);
2942 case RTE_FLOW_ITEM_TYPE_IPV6:
2943 retval = sizeof(struct rte_ipv6_hdr);
2945 case RTE_FLOW_ITEM_TYPE_UDP:
2946 retval = sizeof(struct rte_udp_hdr);
2948 case RTE_FLOW_ITEM_TYPE_TCP:
2949 retval = sizeof(struct rte_tcp_hdr);
2951 case RTE_FLOW_ITEM_TYPE_VXLAN:
2952 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2953 retval = sizeof(struct rte_vxlan_hdr);
2955 case RTE_FLOW_ITEM_TYPE_GRE:
2956 case RTE_FLOW_ITEM_TYPE_NVGRE:
2957 retval = sizeof(struct rte_gre_hdr);
2959 case RTE_FLOW_ITEM_TYPE_MPLS:
2960 retval = sizeof(struct rte_mpls_hdr);
2962 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2970 #define MLX5_ENCAP_IPV4_VERSION 0x40
2971 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
2972 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
2973 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
2974 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
2975 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
2976 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
2979 * Convert the encap action data from list of rte_flow_item to raw buffer
2982 * Pointer to rte_flow_item objects list.
2984 * Pointer to the output buffer.
2986 * Pointer to the output buffer size.
2988 * Pointer to the error structure.
2991 * 0 on success, a negative errno value otherwise and rte_errno is set.
2994 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2995 size_t *size, struct rte_flow_error *error)
2997 struct rte_ether_hdr *eth = NULL;
2998 struct rte_vlan_hdr *vlan = NULL;
2999 struct rte_ipv4_hdr *ipv4 = NULL;
3000 struct rte_ipv6_hdr *ipv6 = NULL;
3001 struct rte_udp_hdr *udp = NULL;
3002 struct rte_vxlan_hdr *vxlan = NULL;
3003 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3004 struct rte_gre_hdr *gre = NULL;
3006 size_t temp_size = 0;
3009 return rte_flow_error_set(error, EINVAL,
3010 RTE_FLOW_ERROR_TYPE_ACTION,
3011 NULL, "invalid empty data");
3012 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3013 len = flow_dv_get_item_hdr_len(items->type);
3014 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3015 return rte_flow_error_set(error, EINVAL,
3016 RTE_FLOW_ERROR_TYPE_ACTION,
3017 (void *)items->type,
3018 "items total size is too big"
3019 " for encap action");
3020 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3021 switch (items->type) {
3022 case RTE_FLOW_ITEM_TYPE_ETH:
3023 eth = (struct rte_ether_hdr *)&buf[temp_size];
3025 case RTE_FLOW_ITEM_TYPE_VLAN:
3026 vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3028 return rte_flow_error_set(error, EINVAL,
3029 RTE_FLOW_ERROR_TYPE_ACTION,
3030 (void *)items->type,
3031 "eth header not found");
3032 if (!eth->ether_type)
3033 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3035 case RTE_FLOW_ITEM_TYPE_IPV4:
3036 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3038 return rte_flow_error_set(error, EINVAL,
3039 RTE_FLOW_ERROR_TYPE_ACTION,
3040 (void *)items->type,
3041 "neither eth nor vlan"
3043 if (vlan && !vlan->eth_proto)
3044 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3045 else if (eth && !eth->ether_type)
3046 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3047 if (!ipv4->version_ihl)
3048 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3049 MLX5_ENCAP_IPV4_IHL_MIN;
3050 if (!ipv4->time_to_live)
3051 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3053 case RTE_FLOW_ITEM_TYPE_IPV6:
3054 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3056 return rte_flow_error_set(error, EINVAL,
3057 RTE_FLOW_ERROR_TYPE_ACTION,
3058 (void *)items->type,
3059 "neither eth nor vlan"
3061 if (vlan && !vlan->eth_proto)
3062 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3063 else if (eth && !eth->ether_type)
3064 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3065 if (!ipv6->vtc_flow)
3067 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3068 if (!ipv6->hop_limits)
3069 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3071 case RTE_FLOW_ITEM_TYPE_UDP:
3072 udp = (struct rte_udp_hdr *)&buf[temp_size];
3074 return rte_flow_error_set(error, EINVAL,
3075 RTE_FLOW_ERROR_TYPE_ACTION,
3076 (void *)items->type,
3077 "ip header not found");
3078 if (ipv4 && !ipv4->next_proto_id)
3079 ipv4->next_proto_id = IPPROTO_UDP;
3080 else if (ipv6 && !ipv6->proto)
3081 ipv6->proto = IPPROTO_UDP;
3083 case RTE_FLOW_ITEM_TYPE_VXLAN:
3084 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3086 return rte_flow_error_set(error, EINVAL,
3087 RTE_FLOW_ERROR_TYPE_ACTION,
3088 (void *)items->type,
3089 "udp header not found");
3091 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3092 if (!vxlan->vx_flags)
3094 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3096 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3097 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3099 return rte_flow_error_set(error, EINVAL,
3100 RTE_FLOW_ERROR_TYPE_ACTION,
3101 (void *)items->type,
3102 "udp header not found");
3103 if (!vxlan_gpe->proto)
3104 return rte_flow_error_set(error, EINVAL,
3105 RTE_FLOW_ERROR_TYPE_ACTION,
3106 (void *)items->type,
3107 "next protocol not found");
3110 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3111 if (!vxlan_gpe->vx_flags)
3112 vxlan_gpe->vx_flags =
3113 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3115 case RTE_FLOW_ITEM_TYPE_GRE:
3116 case RTE_FLOW_ITEM_TYPE_NVGRE:
3117 gre = (struct rte_gre_hdr *)&buf[temp_size];
3119 return rte_flow_error_set(error, EINVAL,
3120 RTE_FLOW_ERROR_TYPE_ACTION,
3121 (void *)items->type,
3122 "next protocol not found");
3124 return rte_flow_error_set(error, EINVAL,
3125 RTE_FLOW_ERROR_TYPE_ACTION,
3126 (void *)items->type,
3127 "ip header not found");
3128 if (ipv4 && !ipv4->next_proto_id)
3129 ipv4->next_proto_id = IPPROTO_GRE;
3130 else if (ipv6 && !ipv6->proto)
3131 ipv6->proto = IPPROTO_GRE;
3133 case RTE_FLOW_ITEM_TYPE_VOID:
3136 return rte_flow_error_set(error, EINVAL,
3137 RTE_FLOW_ERROR_TYPE_ACTION,
3138 (void *)items->type,
3139 "unsupported item type");
3149 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3151 struct rte_ether_hdr *eth = NULL;
3152 struct rte_vlan_hdr *vlan = NULL;
3153 struct rte_ipv6_hdr *ipv6 = NULL;
3154 struct rte_udp_hdr *udp = NULL;
3158 eth = (struct rte_ether_hdr *)data;
3159 next_hdr = (char *)(eth + 1);
3160 proto = RTE_BE16(eth->ether_type);
3163 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3164 vlan = (struct rte_vlan_hdr *)next_hdr;
3165 proto = RTE_BE16(vlan->eth_proto);
3166 next_hdr += sizeof(struct rte_vlan_hdr);
3169 /* HW calculates IPv4 csum. no need to proceed */
3170 if (proto == RTE_ETHER_TYPE_IPV4)
3173 /* non IPv4/IPv6 header. not supported */
3174 if (proto != RTE_ETHER_TYPE_IPV6) {
3175 return rte_flow_error_set(error, ENOTSUP,
3176 RTE_FLOW_ERROR_TYPE_ACTION,
3177 NULL, "Cannot offload non IPv4/IPv6");
3180 ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3182 /* ignore non UDP */
3183 if (ipv6->proto != IPPROTO_UDP)
3186 udp = (struct rte_udp_hdr *)(ipv6 + 1);
3187 udp->dgram_cksum = 0;
3193 * Convert L2 encap action to DV specification.
3196 * Pointer to rte_eth_dev structure.
3198 * Pointer to action structure.
3199 * @param[in, out] dev_flow
3200 * Pointer to the mlx5_flow.
3201 * @param[in] transfer
3202 * Mark if the flow is E-Switch flow.
3204 * Pointer to the error structure.
3207 * 0 on success, a negative errno value otherwise and rte_errno is set.
3210 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3211 const struct rte_flow_action *action,
3212 struct mlx5_flow *dev_flow,
3214 struct rte_flow_error *error)
3216 const struct rte_flow_item *encap_data;
3217 const struct rte_flow_action_raw_encap *raw_encap_data;
3218 struct mlx5_flow_dv_encap_decap_resource res = {
3220 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3221 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3222 MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3225 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3227 (const struct rte_flow_action_raw_encap *)action->conf;
3228 res.size = raw_encap_data->size;
3229 memcpy(res.buf, raw_encap_data->data, res.size);
3231 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3233 ((const struct rte_flow_action_vxlan_encap *)
3234 action->conf)->definition;
3237 ((const struct rte_flow_action_nvgre_encap *)
3238 action->conf)->definition;
3239 if (flow_dv_convert_encap_data(encap_data, res.buf,
3243 if (flow_dv_zero_encap_udp_csum(res.buf, error))
3245 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3246 return rte_flow_error_set(error, EINVAL,
3247 RTE_FLOW_ERROR_TYPE_ACTION,
3248 NULL, "can't create L2 encap action");
3253 * Convert L2 decap action to DV specification.
3256 * Pointer to rte_eth_dev structure.
3257 * @param[in, out] dev_flow
3258 * Pointer to the mlx5_flow.
3259 * @param[in] transfer
3260 * Mark if the flow is E-Switch flow.
3262 * Pointer to the error structure.
3265 * 0 on success, a negative errno value otherwise and rte_errno is set.
3268 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3269 struct mlx5_flow *dev_flow,
3271 struct rte_flow_error *error)
3273 struct mlx5_flow_dv_encap_decap_resource res = {
3276 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3277 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3278 MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3281 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3282 return rte_flow_error_set(error, EINVAL,
3283 RTE_FLOW_ERROR_TYPE_ACTION,
3284 NULL, "can't create L2 decap action");
3289 * Convert raw decap/encap (L3 tunnel) action to DV specification.
3292 * Pointer to rte_eth_dev structure.
3294 * Pointer to action structure.
3295 * @param[in, out] dev_flow
3296 * Pointer to the mlx5_flow.
3298 * Pointer to the flow attributes.
3300 * Pointer to the error structure.
3303 * 0 on success, a negative errno value otherwise and rte_errno is set.
3306 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3307 const struct rte_flow_action *action,
3308 struct mlx5_flow *dev_flow,
3309 const struct rte_flow_attr *attr,
3310 struct rte_flow_error *error)
3312 const struct rte_flow_action_raw_encap *encap_data;
3313 struct mlx5_flow_dv_encap_decap_resource res;
3315 memset(&res, 0, sizeof(res));
3316 encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3317 res.size = encap_data->size;
3318 memcpy(res.buf, encap_data->data, res.size);
3319 res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3320 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3321 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3323 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3325 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3326 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3327 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3328 return rte_flow_error_set(error, EINVAL,
3329 RTE_FLOW_ERROR_TYPE_ACTION,
3330 NULL, "can't create encap action");
3335 * Create action push VLAN.
3338 * Pointer to rte_eth_dev structure.
3340 * Pointer to the flow attributes.
3342 * Pointer to the vlan to push to the Ethernet header.
3343 * @param[in, out] dev_flow
3344 * Pointer to the mlx5_flow.
3346 * Pointer to the error structure.
3349 * 0 on success, a negative errno value otherwise and rte_errno is set.
3352 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3353 const struct rte_flow_attr *attr,
3354 const struct rte_vlan_hdr *vlan,
3355 struct mlx5_flow *dev_flow,
3356 struct rte_flow_error *error)
3358 struct mlx5_flow_dv_push_vlan_action_resource res;
3360 memset(&res, 0, sizeof(res));
3362 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3365 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3367 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3368 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3369 return flow_dv_push_vlan_action_resource_register
3370 (dev, &res, dev_flow, error);
3374 * Validate the modify-header actions.
3376 * @param[in] action_flags
3377 * Holds the actions detected until now.
3379 * Pointer to the modify action.
3381 * Pointer to error structure.
3384 * 0 on success, a negative errno value otherwise and rte_errno is set.
3387 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3388 const struct rte_flow_action *action,
3389 struct rte_flow_error *error)
3391 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3392 return rte_flow_error_set(error, EINVAL,
3393 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3394 NULL, "action configuration not set");
3395 if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3396 return rte_flow_error_set(error, EINVAL,
3397 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3398 "can't have encap action before"
3404 * Validate the modify-header MAC address actions.
3406 * @param[in] action_flags
3407 * Holds the actions detected until now.
3409 * Pointer to the modify action.
3410 * @param[in] item_flags
3411 * Holds the items detected.
3413 * Pointer to error structure.
3416 * 0 on success, a negative errno value otherwise and rte_errno is set.
3419 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3420 const struct rte_flow_action *action,
3421 const uint64_t item_flags,
3422 struct rte_flow_error *error)
3426 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3428 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3429 return rte_flow_error_set(error, EINVAL,
3430 RTE_FLOW_ERROR_TYPE_ACTION,
3432 "no L2 item in pattern");
3438 * Validate the modify-header IPv4 address actions.
3440 * @param[in] action_flags
3441 * Holds the actions detected until now.
3443 * Pointer to the modify action.
3444 * @param[in] item_flags
3445 * Holds the items detected.
3447 * Pointer to error structure.
3450 * 0 on success, a negative errno value otherwise and rte_errno is set.
3453 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3454 const struct rte_flow_action *action,
3455 const uint64_t item_flags,
3456 struct rte_flow_error *error)
3461 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3463 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3464 MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3465 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3466 if (!(item_flags & layer))
3467 return rte_flow_error_set(error, EINVAL,
3468 RTE_FLOW_ERROR_TYPE_ACTION,
3470 "no ipv4 item in pattern");
3476 * Validate the modify-header IPv6 address actions.
3478 * @param[in] action_flags
3479 * Holds the actions detected until now.
3481 * Pointer to the modify action.
3482 * @param[in] item_flags
3483 * Holds the items detected.
3485 * Pointer to error structure.
3488 * 0 on success, a negative errno value otherwise and rte_errno is set.
3491 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3492 const struct rte_flow_action *action,
3493 const uint64_t item_flags,
3494 struct rte_flow_error *error)
3499 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3501 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3502 MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3503 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3504 if (!(item_flags & layer))
3505 return rte_flow_error_set(error, EINVAL,
3506 RTE_FLOW_ERROR_TYPE_ACTION,
3508 "no ipv6 item in pattern");
3514 * Validate the modify-header TP actions.
3516 * @param[in] action_flags
3517 * Holds the actions detected until now.
3519 * Pointer to the modify action.
3520 * @param[in] item_flags
3521 * Holds the items detected.
3523 * Pointer to error structure.
3526 * 0 on success, a negative errno value otherwise and rte_errno is set.
3529 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3530 const struct rte_flow_action *action,
3531 const uint64_t item_flags,
3532 struct rte_flow_error *error)
3537 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3539 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3540 MLX5_FLOW_LAYER_INNER_L4 :
3541 MLX5_FLOW_LAYER_OUTER_L4;
3542 if (!(item_flags & layer))
3543 return rte_flow_error_set(error, EINVAL,
3544 RTE_FLOW_ERROR_TYPE_ACTION,
3545 NULL, "no transport layer "
3552 * Validate the modify-header actions of increment/decrement
3553 * TCP Sequence-number.
3555 * @param[in] action_flags
3556 * Holds the actions detected until now.
3558 * Pointer to the modify action.
3559 * @param[in] item_flags
3560 * Holds the items detected.
3562 * Pointer to error structure.
3565 * 0 on success, a negative errno value otherwise and rte_errno is set.
3568 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3569 const struct rte_flow_action *action,
3570 const uint64_t item_flags,
3571 struct rte_flow_error *error)
3576 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3578 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3579 MLX5_FLOW_LAYER_INNER_L4_TCP :
3580 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3581 if (!(item_flags & layer))
3582 return rte_flow_error_set(error, EINVAL,
3583 RTE_FLOW_ERROR_TYPE_ACTION,
3584 NULL, "no TCP item in"
3586 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3587 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3588 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3589 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3590 return rte_flow_error_set(error, EINVAL,
3591 RTE_FLOW_ERROR_TYPE_ACTION,
3593 "cannot decrease and increase"
3594 " TCP sequence number"
3595 " at the same time");
3601 * Validate the modify-header actions of increment/decrement
3602 * TCP Acknowledgment number.
3604 * @param[in] action_flags
3605 * Holds the actions detected until now.
3607 * Pointer to the modify action.
3608 * @param[in] item_flags
3609 * Holds the items detected.
3611 * Pointer to error structure.
3614 * 0 on success, a negative errno value otherwise and rte_errno is set.
3617 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3618 const struct rte_flow_action *action,
3619 const uint64_t item_flags,
3620 struct rte_flow_error *error)
3625 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3627 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3628 MLX5_FLOW_LAYER_INNER_L4_TCP :
3629 MLX5_FLOW_LAYER_OUTER_L4_TCP;
3630 if (!(item_flags & layer))
3631 return rte_flow_error_set(error, EINVAL,
3632 RTE_FLOW_ERROR_TYPE_ACTION,
3633 NULL, "no TCP item in"
3635 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3636 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3637 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3638 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3639 return rte_flow_error_set(error, EINVAL,
3640 RTE_FLOW_ERROR_TYPE_ACTION,
3642 "cannot decrease and increase"
3643 " TCP acknowledgment number"
3644 " at the same time");
3650 * Validate the modify-header TTL actions.
3652 * @param[in] action_flags
3653 * Holds the actions detected until now.
3655 * Pointer to the modify action.
3656 * @param[in] item_flags
3657 * Holds the items detected.
3659 * Pointer to error structure.
3662 * 0 on success, a negative errno value otherwise and rte_errno is set.
3665 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3666 const struct rte_flow_action *action,
3667 const uint64_t item_flags,
3668 struct rte_flow_error *error)
3673 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3675 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3676 MLX5_FLOW_LAYER_INNER_L3 :
3677 MLX5_FLOW_LAYER_OUTER_L3;
3678 if (!(item_flags & layer))
3679 return rte_flow_error_set(error, EINVAL,
3680 RTE_FLOW_ERROR_TYPE_ACTION,
3682 "no IP protocol in pattern");
3688 * Validate jump action.
3691 * Pointer to the jump action.
3692 * @param[in] action_flags
3693 * Holds the actions detected until now.
3694 * @param[in] attributes
3695 * Pointer to flow attributes
3696 * @param[in] external
3697 * Action belongs to flow rule created by request external to PMD.
3699 * Pointer to error structure.
3702 * 0 on success, a negative errno value otherwise and rte_errno is set.
3705 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3706 uint64_t action_flags,
3707 const struct rte_flow_attr *attributes,
3708 bool external, struct rte_flow_error *error)
3710 uint32_t target_group, table;
3713 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3714 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3715 return rte_flow_error_set(error, EINVAL,
3716 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3717 "can't have 2 fate actions in"
3719 if (action_flags & MLX5_FLOW_ACTION_METER)
3720 return rte_flow_error_set(error, ENOTSUP,
3721 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3722 "jump with meter not support");
3724 return rte_flow_error_set(error, EINVAL,
3725 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3726 NULL, "action configuration not set");
3728 ((const struct rte_flow_action_jump *)action->conf)->group;
3729 ret = mlx5_flow_group_to_table(attributes, external, target_group,
3730 true, &table, error);
3733 if (attributes->group == target_group)
3734 return rte_flow_error_set(error, EINVAL,
3735 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3736 "target group must be other than"
3737 " the current flow group");
3742 * Validate the port_id action.
3745 * Pointer to rte_eth_dev structure.
3746 * @param[in] action_flags
3747 * Bit-fields that holds the actions detected until now.
3749 * Port_id RTE action structure.
3751 * Attributes of flow that includes this action.
3753 * Pointer to error structure.
3756 * 0 on success, a negative errno value otherwise and rte_errno is set.
3759 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3760 uint64_t action_flags,
3761 const struct rte_flow_action *action,
3762 const struct rte_flow_attr *attr,
3763 struct rte_flow_error *error)
3765 const struct rte_flow_action_port_id *port_id;
3766 struct mlx5_priv *act_priv;
3767 struct mlx5_priv *dev_priv;
3770 if (!attr->transfer)
3771 return rte_flow_error_set(error, ENOTSUP,
3772 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3774 "port id action is valid in transfer"
3776 if (!action || !action->conf)
3777 return rte_flow_error_set(error, ENOTSUP,
3778 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3780 "port id action parameters must be"
3782 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3783 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3784 return rte_flow_error_set(error, EINVAL,
3785 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3786 "can have only one fate actions in"
3788 dev_priv = mlx5_dev_to_eswitch_info(dev);
3790 return rte_flow_error_set(error, rte_errno,
3791 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3793 "failed to obtain E-Switch info");
3794 port_id = action->conf;
3795 port = port_id->original ? dev->data->port_id : port_id->id;
3796 act_priv = mlx5_port_to_eswitch_info(port, false);
3798 return rte_flow_error_set
3800 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3801 "failed to obtain E-Switch port id for port");
3802 if (act_priv->domain_id != dev_priv->domain_id)
3803 return rte_flow_error_set
3805 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3806 "port does not belong to"
3807 " E-Switch being configured");
3812 * Get the maximum number of modify header actions.
3815 * Pointer to rte_eth_dev structure.
3817 * Flags bits to check if root level.
3820 * Max number of modify header actions device can support.
3822 static inline unsigned int
3823 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3827 * There's no way to directly query the max capacity from FW.
3828 * The maximal value on root table should be assumed to be supported.
3830 if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3831 return MLX5_MAX_MODIFY_NUM;
3833 return MLX5_ROOT_TBL_MODIFY_NUM;
3837 * Validate the meter action.
3840 * Pointer to rte_eth_dev structure.
3841 * @param[in] action_flags
3842 * Bit-fields that holds the actions detected until now.
3844 * Pointer to the meter action.
3846 * Attributes of flow that includes this action.
3848 * Pointer to error structure.
3851 * 0 on success, a negative errno value otherwise and rte_ernno is set.
3854 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3855 uint64_t action_flags,
3856 const struct rte_flow_action *action,
3857 const struct rte_flow_attr *attr,
3858 struct rte_flow_error *error)
3860 struct mlx5_priv *priv = dev->data->dev_private;
3861 const struct rte_flow_action_meter *am = action->conf;
3862 struct mlx5_flow_meter *fm;
3865 return rte_flow_error_set(error, EINVAL,
3866 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3867 "meter action conf is NULL");
3869 if (action_flags & MLX5_FLOW_ACTION_METER)
3870 return rte_flow_error_set(error, ENOTSUP,
3871 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3872 "meter chaining not support");
3873 if (action_flags & MLX5_FLOW_ACTION_JUMP)
3874 return rte_flow_error_set(error, ENOTSUP,
3875 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3876 "meter with jump not support");
3878 return rte_flow_error_set(error, ENOTSUP,
3879 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3881 "meter action not supported");
3882 fm = mlx5_flow_meter_find(priv, am->mtr_id);
3884 return rte_flow_error_set(error, EINVAL,
3885 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3887 if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3888 (!fm->ingress && !attr->ingress && attr->egress) ||
3889 (!fm->egress && !attr->egress && attr->ingress))))
3890 return rte_flow_error_set(error, EINVAL,
3891 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3892 "Flow attributes are either invalid "
3893 "or have a conflict with current "
3894 "meter attributes");
3899 * Validate the age action.
3901 * @param[in] action_flags
3902 * Holds the actions detected until now.
3904 * Pointer to the age action.
3906 * Pointer to the Ethernet device structure.
3908 * Pointer to error structure.
3911 * 0 on success, a negative errno value otherwise and rte_errno is set.
3914 flow_dv_validate_action_age(uint64_t action_flags,
3915 const struct rte_flow_action *action,
3916 struct rte_eth_dev *dev,
3917 struct rte_flow_error *error)
3919 struct mlx5_priv *priv = dev->data->dev_private;
3920 const struct rte_flow_action_age *age = action->conf;
3922 if (!priv->config.devx || priv->counter_fallback)
3923 return rte_flow_error_set(error, ENOTSUP,
3924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3926 "age action not supported");
3927 if (!(action->conf))
3928 return rte_flow_error_set(error, EINVAL,
3929 RTE_FLOW_ERROR_TYPE_ACTION, action,
3930 "configuration cannot be null");
3931 if (age->timeout >= UINT16_MAX / 2 / 10)
3932 return rte_flow_error_set(error, ENOTSUP,
3933 RTE_FLOW_ERROR_TYPE_ACTION, action,
3934 "Max age time: 3275 seconds");
3935 if (action_flags & MLX5_FLOW_ACTION_AGE)
3936 return rte_flow_error_set(error, EINVAL,
3937 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3938 "Duplicate age ctions set");
3943 * Validate the modify-header IPv4 DSCP actions.
3945 * @param[in] action_flags
3946 * Holds the actions detected until now.
3948 * Pointer to the modify action.
3949 * @param[in] item_flags
3950 * Holds the items detected.
3952 * Pointer to error structure.
3955 * 0 on success, a negative errno value otherwise and rte_errno is set.
3958 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3959 const struct rte_flow_action *action,
3960 const uint64_t item_flags,
3961 struct rte_flow_error *error)
3965 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3967 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3968 return rte_flow_error_set(error, EINVAL,
3969 RTE_FLOW_ERROR_TYPE_ACTION,
3971 "no ipv4 item in pattern");
3977 * Validate the modify-header IPv6 DSCP actions.
3979 * @param[in] action_flags
3980 * Holds the actions detected until now.
3982 * Pointer to the modify action.
3983 * @param[in] item_flags
3984 * Holds the items detected.
3986 * Pointer to error structure.
3989 * 0 on success, a negative errno value otherwise and rte_errno is set.
3992 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3993 const struct rte_flow_action *action,
3994 const uint64_t item_flags,
3995 struct rte_flow_error *error)
3999 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4001 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4002 return rte_flow_error_set(error, EINVAL,
4003 RTE_FLOW_ERROR_TYPE_ACTION,
4005 "no ipv6 item in pattern");
4011 * Match modify-header resource.
4014 * Pointer to exist resource entry object.
4016 * Pointer to new modify-header resource.
4019 * 0 on matching, -1 otherwise.
4022 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4024 struct mlx5_flow_dv_modify_hdr_resource *resource;
4025 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4026 uint32_t actions_len;
4028 resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4029 cache_resource = container_of(entry,
4030 struct mlx5_flow_dv_modify_hdr_resource,
4032 actions_len = resource->actions_num * sizeof(resource->actions[0]);
4033 if (resource->entry.key == cache_resource->entry.key &&
4034 resource->ft_type == cache_resource->ft_type &&
4035 resource->actions_num == cache_resource->actions_num &&
4036 resource->flags == cache_resource->flags &&
4037 !memcmp((const void *)resource->actions,
4038 (const void *)cache_resource->actions,
4045 * Validate the sample action.
4047 * @param[in] action_flags
4048 * Holds the actions detected until now.
4050 * Pointer to the sample action.
4052 * Pointer to the Ethernet device structure.
4054 * Attributes of flow that includes this action.
4056 * Pointer to error structure.
4059 * 0 on success, a negative errno value otherwise and rte_errno is set.
4062 flow_dv_validate_action_sample(uint64_t action_flags,
4063 const struct rte_flow_action *action,
4064 struct rte_eth_dev *dev,
4065 const struct rte_flow_attr *attr,
4066 struct rte_flow_error *error)
4068 struct mlx5_priv *priv = dev->data->dev_private;
4069 struct mlx5_dev_config *dev_conf = &priv->config;
4070 const struct rte_flow_action_sample *sample = action->conf;
4071 const struct rte_flow_action *act;
4072 uint64_t sub_action_flags = 0;
4077 return rte_flow_error_set(error, EINVAL,
4078 RTE_FLOW_ERROR_TYPE_ACTION, action,
4079 "configuration cannot be NULL");
4080 if (sample->ratio == 0)
4081 return rte_flow_error_set(error, EINVAL,
4082 RTE_FLOW_ERROR_TYPE_ACTION, action,
4083 "ratio value starts from 1");
4084 if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4085 return rte_flow_error_set(error, ENOTSUP,
4086 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4088 "sample action not supported");
4089 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4090 return rte_flow_error_set(error, EINVAL,
4091 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4092 "Multiple sample actions not "
4094 if (action_flags & MLX5_FLOW_ACTION_METER)
4095 return rte_flow_error_set(error, EINVAL,
4096 RTE_FLOW_ERROR_TYPE_ACTION, action,
4097 "wrong action order, meter should "
4098 "be after sample action");
4099 if (action_flags & MLX5_FLOW_ACTION_JUMP)
4100 return rte_flow_error_set(error, EINVAL,
4101 RTE_FLOW_ERROR_TYPE_ACTION, action,
4102 "wrong action order, jump should "
4103 "be after sample action");
4104 act = sample->actions;
4105 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4106 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4107 return rte_flow_error_set(error, ENOTSUP,
4108 RTE_FLOW_ERROR_TYPE_ACTION,
4109 act, "too many actions");
4110 switch (act->type) {
4111 case RTE_FLOW_ACTION_TYPE_QUEUE:
4112 ret = mlx5_flow_validate_action_queue(act,
4118 sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4121 case RTE_FLOW_ACTION_TYPE_MARK:
4122 ret = flow_dv_validate_action_mark(dev, act,
4127 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4128 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4129 MLX5_FLOW_ACTION_MARK_EXT;
4131 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4134 case RTE_FLOW_ACTION_TYPE_COUNT:
4135 ret = flow_dv_validate_action_count(dev, error);
4138 sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4142 return rte_flow_error_set(error, ENOTSUP,
4143 RTE_FLOW_ERROR_TYPE_ACTION,
4145 "Doesn't support optional "
4149 if (attr->ingress && !attr->transfer) {
4150 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4151 return rte_flow_error_set(error, EINVAL,
4152 RTE_FLOW_ERROR_TYPE_ACTION,
4154 "Ingress must has a dest "
4155 "QUEUE for Sample");
4156 } else if (attr->egress && !attr->transfer) {
4157 return rte_flow_error_set(error, ENOTSUP,
4158 RTE_FLOW_ERROR_TYPE_ACTION,
4160 "Sample Only support Ingress "
4162 } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4163 MLX5_ASSERT(attr->transfer);
4164 return rte_flow_error_set(error, ENOTSUP,
4165 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4166 "E-Switch doesn't support any "
4167 "optional action for sampling");
4173 * Find existing modify-header resource or create and register a new one.
4175 * @param dev[in, out]
4176 * Pointer to rte_eth_dev structure.
4177 * @param[in, out] resource
4178 * Pointer to modify-header resource.
4179 * @parm[in, out] dev_flow
4180 * Pointer to the dev_flow.
4182 * pointer to error structure.
4185 * 0 on success otherwise -errno and errno is set.
4188 flow_dv_modify_hdr_resource_register
4189 (struct rte_eth_dev *dev,
4190 struct mlx5_flow_dv_modify_hdr_resource *resource,
4191 struct mlx5_flow *dev_flow,
4192 struct rte_flow_error *error)
4194 struct mlx5_priv *priv = dev->data->dev_private;
4195 struct mlx5_dev_ctx_shared *sh = priv->sh;
4196 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4197 struct mlx5dv_dr_domain *ns;
4198 uint32_t actions_len;
4199 struct mlx5_hlist_entry *entry;
4200 union mlx5_flow_modify_hdr_key hdr_mod_key = {
4202 .ft_type = resource->ft_type,
4203 .actions_num = resource->actions_num,
4204 .group = dev_flow->dv.group,
4210 resource->flags = dev_flow->dv.group ? 0 :
4211 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4212 if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4214 return rte_flow_error_set(error, EOVERFLOW,
4215 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4216 "too many modify header items");
4217 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4218 ns = sh->fdb_domain;
4219 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4223 /* Lookup a matching resource from cache. */
4224 actions_len = resource->actions_num * sizeof(resource->actions[0]);
4225 hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4226 resource->entry.key = hdr_mod_key.v64;
4227 entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4228 flow_dv_modify_hdr_resource_match,
4231 cache_resource = container_of(entry,
4232 struct mlx5_flow_dv_modify_hdr_resource,
4234 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4235 (void *)cache_resource,
4236 rte_atomic32_read(&cache_resource->refcnt));
4237 rte_atomic32_inc(&cache_resource->refcnt);
4238 dev_flow->handle->dvh.modify_hdr = cache_resource;
4242 /* Register new modify-header resource. */
4243 cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4244 sizeof(*cache_resource) + actions_len, 0,
4246 if (!cache_resource)
4247 return rte_flow_error_set(error, ENOMEM,
4248 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4249 "cannot allocate resource memory");
4250 *cache_resource = *resource;
4251 rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4252 ret = mlx5_flow_os_create_flow_action_modify_header
4253 (sh->ctx, ns, cache_resource,
4254 actions_len, &cache_resource->action);
4256 mlx5_free(cache_resource);
4257 return rte_flow_error_set(error, ENOMEM,
4258 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4259 NULL, "cannot create action");
4261 rte_atomic32_init(&cache_resource->refcnt);
4262 rte_atomic32_inc(&cache_resource->refcnt);
4263 if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4264 flow_dv_modify_hdr_resource_match,
4265 (void *)cache_resource)) {
4266 claim_zero(mlx5_flow_os_destroy_flow_action
4267 (cache_resource->action));
4268 mlx5_free(cache_resource);
4269 return rte_flow_error_set(error, EEXIST,
4270 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4271 NULL, "action exist");
4273 dev_flow->handle->dvh.modify_hdr = cache_resource;
4274 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4275 (void *)cache_resource,
4276 rte_atomic32_read(&cache_resource->refcnt));
4281 * Get DV flow counter by index.
4284 * Pointer to the Ethernet device structure.
4286 * mlx5 flow counter index in the container.
4288 * mlx5 flow counter pool in the container,
4291 * Pointer to the counter, NULL otherwise.
4293 static struct mlx5_flow_counter *
4294 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4296 struct mlx5_flow_counter_pool **ppool)
4298 struct mlx5_priv *priv = dev->data->dev_private;
4299 struct mlx5_pools_container *cont;
4300 struct mlx5_flow_counter_pool *pool;
4301 uint32_t batch = 0, age = 0;
4304 age = MLX_CNT_IS_AGE(idx);
4305 idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4306 if (idx >= MLX5_CNT_BATCH_OFFSET) {
4307 idx -= MLX5_CNT_BATCH_OFFSET;
4310 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4311 MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4312 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4316 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4320 * Check the devx counter belongs to the pool.
4323 * Pointer to the counter pool.
4325 * The counter devx ID.
4328 * True if counter belongs to the pool, false otherwise.
4331 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4333 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4334 MLX5_COUNTERS_PER_POOL;
4336 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4342 * Get a pool by devx counter ID.
4345 * Pointer to the counter container.
4347 * The counter devx ID.
4350 * The counter pool pointer if exists, NULL otherwise,
4352 static struct mlx5_flow_counter_pool *
4353 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4357 /* Check last used pool. */
4358 if (cont->last_pool_idx != POOL_IDX_INVALID &&
4359 flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4360 return cont->pools[cont->last_pool_idx];
4361 /* ID out of range means no suitable pool in the container. */
4362 if (id > cont->max_id || id < cont->min_id)
4365 * Find the pool from the end of the container, since mostly counter
4366 * ID is sequence increasing, and the last pool should be the needed
4369 i = rte_atomic16_read(&cont->n_valid);
4371 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4373 if (flow_dv_is_counter_in_pool(pool, id))
4380 * Allocate a new memory for the counter values wrapped by all the needed
4384 * Pointer to the Ethernet device structure.
4386 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4389 * The new memory management pointer on success, otherwise NULL and rte_errno
4392 static struct mlx5_counter_stats_mem_mng *
4393 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4395 struct mlx5_priv *priv = dev->data->dev_private;
4396 struct mlx5_dev_ctx_shared *sh = priv->sh;
4397 struct mlx5_devx_mkey_attr mkey_attr;
4398 struct mlx5_counter_stats_mem_mng *mem_mng;
4399 volatile struct flow_counter_stats *raw_data;
4400 int size = (sizeof(struct flow_counter_stats) *
4401 MLX5_COUNTERS_PER_POOL +
4402 sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4403 sizeof(struct mlx5_counter_stats_mem_mng);
4404 size_t pgsize = rte_mem_page_size();
4405 if (pgsize == (size_t)-1) {
4406 DRV_LOG(ERR, "Failed to get mem page size");
4410 uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
4418 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4419 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4420 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4421 IBV_ACCESS_LOCAL_WRITE);
4422 if (!mem_mng->umem) {
4427 mkey_attr.addr = (uintptr_t)mem;
4428 mkey_attr.size = size;
4429 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4430 mkey_attr.pd = sh->pdn;
4431 mkey_attr.log_entity_size = 0;
4432 mkey_attr.pg_access = 0;
4433 mkey_attr.klm_array = NULL;
4434 mkey_attr.klm_num = 0;
4435 if (priv->config.hca_attr.relaxed_ordering_write &&
4436 priv->config.hca_attr.relaxed_ordering_read &&
4437 !haswell_broadwell_cpu)
4438 mkey_attr.relaxed_ordering = 1;
4439 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4441 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4446 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4447 raw_data = (volatile struct flow_counter_stats *)mem;
4448 for (i = 0; i < raws_n; ++i) {
4449 mem_mng->raws[i].mem_mng = mem_mng;
4450 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4452 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4457 * Resize a counter container.
4460 * Pointer to the Ethernet device structure.
4462 * Whether the pool is for counter that was allocated by batch command.
4464 * Whether the pool is for Aging counter.
4467 * 0 on success, otherwise negative errno value and rte_errno is set.
4470 flow_dv_container_resize(struct rte_eth_dev *dev,
4471 uint32_t batch, uint32_t age)
4473 struct mlx5_priv *priv = dev->data->dev_private;
4474 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4476 struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4477 void *old_pools = cont->pools;
4478 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4479 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4480 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4487 memcpy(pools, old_pools, cont->n *
4488 sizeof(struct mlx5_flow_counter_pool *));
4490 * Fallback mode query the counter directly, no background query
4491 * resources are needed.
4493 if (!priv->counter_fallback) {
4496 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4497 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4502 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4503 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4505 MLX5_CNT_CONTAINER_RESIZE +
4508 rte_spinlock_lock(&cont->resize_sl);
4510 cont->mem_mng = mem_mng;
4511 cont->pools = pools;
4512 rte_spinlock_unlock(&cont->resize_sl);
4514 mlx5_free(old_pools);
4519 * Query a devx flow counter.
4522 * Pointer to the Ethernet device structure.
4524 * Index to the flow counter.
4526 * The statistics value of packets.
4528 * The statistics value of bytes.
4531 * 0 on success, otherwise a negative errno value and rte_errno is set.
4534 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4537 struct mlx5_priv *priv = dev->data->dev_private;
4538 struct mlx5_flow_counter_pool *pool = NULL;
4539 struct mlx5_flow_counter *cnt;
4540 struct mlx5_flow_counter_ext *cnt_ext = NULL;
4543 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4545 if (counter < MLX5_CNT_BATCH_OFFSET) {
4546 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4547 if (priv->counter_fallback)
4548 return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4549 0, pkts, bytes, 0, NULL, NULL, 0);
4552 rte_spinlock_lock(&pool->sl);
4554 * The single counters allocation may allocate smaller ID than the
4555 * current allocated in parallel to the host reading.
4556 * In this case the new counter values must be reported as 0.
4558 if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4562 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4563 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4564 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4566 rte_spinlock_unlock(&pool->sl);
4571 * Create and initialize a new counter pool.
4574 * Pointer to the Ethernet device structure.
4576 * The devX counter handle.
4578 * Whether the pool is for counter that was allocated by batch command.
4580 * Whether the pool is for counter that was allocated for aging.
4581 * @param[in/out] cont_cur
4582 * Pointer to the container pointer, it will be update in pool resize.
4585 * The pool container pointer on success, NULL otherwise and rte_errno is set.
4587 static struct mlx5_flow_counter_pool *
4588 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4589 uint32_t batch, uint32_t age)
4591 struct mlx5_priv *priv = dev->data->dev_private;
4592 struct mlx5_flow_counter_pool *pool;
4593 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4595 int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4596 uint32_t size = sizeof(*pool);
4598 if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4600 size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4601 size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4602 size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4603 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4608 pool->min_dcs = dcs;
4609 if (!priv->counter_fallback)
4610 pool->raw = cont->mem_mng->raws + n_valid %
4611 MLX5_CNT_CONTAINER_RESIZE;
4612 pool->raw_hw = NULL;
4614 pool->type |= (batch ? 0 : CNT_POOL_TYPE_EXT);
4615 pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE);
4616 pool->query_gen = 0;
4617 rte_spinlock_init(&pool->sl);
4618 TAILQ_INIT(&pool->counters[0]);
4619 TAILQ_INIT(&pool->counters[1]);
4620 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4621 pool->index = n_valid;
4622 cont->pools[n_valid] = pool;
4624 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4626 if (base < cont->min_id)
4627 cont->min_id = base;
4628 if (base > cont->max_id)
4629 cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4630 cont->last_pool_idx = pool->index;
4632 /* Pool initialization must be updated before host thread access. */
4634 rte_atomic16_add(&cont->n_valid, 1);
4639 * Restore skipped counters in the pool.
4641 * As counter pool query requires the first counter dcs
4642 * ID start with 4 alinged, if the pool counters with
4643 * min_dcs ID are not aligned with 4, the counters will
4645 * Once other min_dcs ID less than these skipped counter
4646 * dcs ID appears, the skipped counters will be safe to
4648 * Should be called when min_dcs is updated.
4651 * Current counter pool.
4652 * @param[in] last_min_dcs
4656 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool,
4657 struct mlx5_devx_obj *last_min_dcs)
4659 struct mlx5_flow_counter_ext *cnt_ext;
4660 uint32_t offset, new_offset;
4661 uint32_t skip_cnt = 0;
4664 if (!pool->skip_cnt)
4667 * If last min_dcs is not valid. The skipped counter may even after
4668 * last min_dcs, set the offset to the whole pool.
4670 if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4671 offset = MLX5_COUNTERS_PER_POOL;
4673 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL;
4674 new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL;
4676 * Check the counters from 1 to the last_min_dcs range. Counters
4677 * before new min_dcs indicates pool still has skipped counters.
4678 * Counters be skipped after new min_dcs will be ready to use.
4679 * Offset 0 counter must be empty or min_dcs, start from 1.
4681 for (i = 1; i < offset; i++) {
4682 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
4683 if (cnt_ext->skipped) {
4684 if (i > new_offset) {
4685 cnt_ext->skipped = 0;
4687 (&pool->counters[pool->query_gen],
4688 MLX5_POOL_GET_CNT(pool, i), next);
4699 * Prepare a new counter and/or a new counter pool.
4702 * Pointer to the Ethernet device structure.
4703 * @param[out] cnt_free
4704 * Where to put the pointer of a new counter.
4706 * Whether the pool is for counter that was allocated by batch command.
4708 * Whether the pool is for counter that was allocated for aging.
4711 * The counter pool pointer and @p cnt_free is set on success,
4712 * NULL otherwise and rte_errno is set.
4714 static struct mlx5_flow_counter_pool *
4715 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4716 struct mlx5_flow_counter **cnt_free,
4717 uint32_t batch, uint32_t age)
4719 struct mlx5_priv *priv = dev->data->dev_private;
4720 struct mlx5_pools_container *cont;
4721 struct mlx5_flow_counter_pool *pool;
4722 struct mlx5_counters tmp_tq;
4723 struct mlx5_devx_obj *last_min_dcs;
4724 struct mlx5_devx_obj *dcs = NULL;
4725 struct mlx5_flow_counter *cnt;
4729 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4733 /* bulk_bitmap must be 0 for single counter allocation. */
4734 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4737 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4738 /* Check if counter belongs to exist pool ID range. */
4740 pool = flow_dv_find_pool_by_id
4742 (priv->sh, batch, (age ^ 0x1)), dcs->id);
4744 * Pool eixsts, counter will be added to the other
4745 * container, need to reallocate it later.
4750 pool = flow_dv_pool_create(dev, dcs, batch,
4753 mlx5_devx_cmd_destroy(dcs);
4758 if ((dcs->id < pool->min_dcs->id ||
4760 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) &&
4761 !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) {
4763 * Update the pool min_dcs only if current dcs is
4764 * valid and exist min_dcs is not valid or greater
4767 last_min_dcs = pool->min_dcs;
4768 rte_atomic64_set(&pool->a64_dcs,
4769 (int64_t)(uintptr_t)dcs);
4771 * Restore any skipped counters if the new min_dcs
4772 * ID is smaller or min_dcs is not valid.
4774 if (dcs->id < last_min_dcs->id ||
4776 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4777 flow_dv_counter_restore(pool, last_min_dcs);
4779 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4780 cnt = MLX5_POOL_GET_CNT(pool, i);
4782 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4784 * If min_dcs is not valid, it means the new allocated dcs
4785 * also fail to become the valid min_dcs, just skip it.
4786 * Or if min_dcs is valid, and new dcs ID is smaller than
4787 * min_dcs, but not become the min_dcs, also skip it.
4789 if (pool->min_dcs->id &
4790 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) ||
4791 dcs->id < pool->min_dcs->id) {
4792 MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1;
4797 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen],
4804 /* bulk_bitmap is in 128 counters units. */
4805 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4806 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4808 rte_errno = ENODATA;
4811 pool = flow_dv_pool_create(dev, dcs, batch, age);
4813 mlx5_devx_cmd_destroy(dcs);
4816 TAILQ_INIT(&tmp_tq);
4817 for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4818 cnt = MLX5_POOL_GET_CNT(pool, i);
4820 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4822 rte_spinlock_lock(&cont->csl);
4823 TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4824 rte_spinlock_unlock(&cont->csl);
4825 *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4826 (*cnt_free)->pool = pool;
4831 * Search for existed shared counter.
4834 * Pointer to the Ethernet device structure.
4836 * The shared counter ID to search.
4838 * mlx5 flow counter pool in the container,
4841 * NULL if not existed, otherwise pointer to the shared extend counter.
4843 static struct mlx5_flow_counter_ext *
4844 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4845 struct mlx5_flow_counter_pool **ppool)
4847 struct mlx5_priv *priv = dev->data->dev_private;
4848 union mlx5_l3t_data data;
4851 if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4853 cnt_idx = data.dword;
4855 * Shared counters don't have age info. The counter extend is after
4856 * the counter datat structure.
4858 return (struct mlx5_flow_counter_ext *)
4859 ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4863 * Allocate a flow counter.
4866 * Pointer to the Ethernet device structure.
4868 * Indicate if this counter is shared with other flows.
4870 * Counter identifier.
4872 * Counter flow group.
4874 * Whether the counter was allocated for aging.
4877 * Index to flow counter on success, 0 otherwise and rte_errno is set.
4880 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4881 uint16_t group, uint32_t age)
4883 struct mlx5_priv *priv = dev->data->dev_private;
4884 struct mlx5_flow_counter_pool *pool = NULL;
4885 struct mlx5_flow_counter *cnt_free = NULL;
4886 struct mlx5_flow_counter_ext *cnt_ext = NULL;
4888 * Currently group 0 flow counter cannot be assigned to a flow if it is
4889 * not the first one in the batch counter allocation, so it is better
4890 * to allocate counters one by one for these flows in a separate
4892 * A counter can be shared between different groups so need to take
4893 * shared counters from the single container.
4895 uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4896 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4900 if (!priv->config.devx) {
4901 rte_errno = ENOTSUP;
4905 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4907 if (cnt_ext->ref_cnt + 1 == 0) {
4912 cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4913 (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4918 /* Get free counters from container. */
4919 rte_spinlock_lock(&cont->csl);
4920 cnt_free = TAILQ_FIRST(&cont->counters);
4922 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4923 rte_spinlock_unlock(&cont->csl);
4924 if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4927 pool = cnt_free->pool;
4929 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4930 /* Create a DV counter action only in the first time usage. */
4931 if (!cnt_free->action) {
4933 struct mlx5_devx_obj *dcs;
4937 offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4938 dcs = pool->min_dcs;
4943 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4950 cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4951 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4952 cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4953 cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4954 /* Update the counter reset values. */
4955 if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4959 cnt_ext->shared = shared;
4960 cnt_ext->ref_cnt = 1;
4963 union mlx5_l3t_data data;
4965 data.dword = cnt_idx;
4966 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4970 if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4971 /* Start the asynchronous batch query by the host thread. */
4972 mlx5_set_query_alarm(priv->sh);
4976 cnt_free->pool = pool;
4977 rte_spinlock_lock(&cont->csl);
4978 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4979 rte_spinlock_unlock(&cont->csl);
4985 * Get age param from counter index.
4988 * Pointer to the Ethernet device structure.
4989 * @param[in] counter
4990 * Index to the counter handler.
4993 * The aging parameter specified for the counter index.
4995 static struct mlx5_age_param*
4996 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4999 struct mlx5_flow_counter *cnt;
5000 struct mlx5_flow_counter_pool *pool = NULL;
5002 flow_dv_counter_get_by_idx(dev, counter, &pool);
5003 counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5004 cnt = MLX5_POOL_GET_CNT(pool, counter);
5005 return MLX5_CNT_TO_AGE(cnt);
5009 * Remove a flow counter from aged counter list.
5012 * Pointer to the Ethernet device structure.
5013 * @param[in] counter
5014 * Index to the counter handler.
5016 * Pointer to the counter handler.
5019 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5020 uint32_t counter, struct mlx5_flow_counter *cnt)
5022 struct mlx5_age_info *age_info;
5023 struct mlx5_age_param *age_param;
5024 struct mlx5_priv *priv = dev->data->dev_private;
5026 age_info = GET_PORT_AGE_INFO(priv);
5027 age_param = flow_dv_counter_idx_get_age(dev, counter);
5028 if (rte_atomic16_cmpset((volatile uint16_t *)
5030 AGE_CANDIDATE, AGE_FREE)
5033 * We need the lock even it is age timeout,
5034 * since counter may still in process.
5036 rte_spinlock_lock(&age_info->aged_sl);
5037 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5038 rte_spinlock_unlock(&age_info->aged_sl);
5040 rte_atomic16_set(&age_param->state, AGE_FREE);
5043 * Release a flow counter.
5046 * Pointer to the Ethernet device structure.
5047 * @param[in] counter
5048 * Index to the counter handler.
5051 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5053 struct mlx5_priv *priv = dev->data->dev_private;
5054 struct mlx5_flow_counter_pool *pool = NULL;
5055 struct mlx5_flow_counter *cnt;
5056 struct mlx5_flow_counter_ext *cnt_ext = NULL;
5060 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5062 if (counter < MLX5_CNT_BATCH_OFFSET) {
5063 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
5065 if (--cnt_ext->ref_cnt)
5067 if (cnt_ext->shared)
5068 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
5072 if (IS_AGE_POOL(pool))
5073 flow_dv_counter_remove_from_age(dev, counter, cnt);
5076 * Put the counter back to list to be updated in none fallback mode.
5077 * Currently, we are using two list alternately, while one is in query,
5078 * add the freed counter to the other list based on the pool query_gen
5079 * value. After query finishes, add counter the list to the global
5080 * container counter list. The list changes while query starts. In
5081 * this case, lock will not be needed as query callback and release
5082 * function both operate with the different list.
5085 if (!priv->counter_fallback)
5086 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5088 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
5089 (priv->sh, 0, 0))->counters),
5094 * Verify the @p attributes will be correctly understood by the NIC and store
5095 * them in the @p flow if everything is correct.
5098 * Pointer to dev struct.
5099 * @param[in] attributes
5100 * Pointer to flow attributes
5101 * @param[in] external
5102 * This flow rule is created by request external to PMD.
5104 * Pointer to error structure.
5107 * - 0 on success and non root table.
5108 * - 1 on success and root table.
5109 * - a negative errno value otherwise and rte_errno is set.
5112 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5113 const struct rte_flow_attr *attributes,
5114 bool external __rte_unused,
5115 struct rte_flow_error *error)
5117 struct mlx5_priv *priv = dev->data->dev_private;
5118 uint32_t priority_max = priv->config.flow_prio - 1;
5121 #ifndef HAVE_MLX5DV_DR
5122 if (attributes->group)
5123 return rte_flow_error_set(error, ENOTSUP,
5124 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5126 "groups are not supported");
5130 ret = mlx5_flow_group_to_table(attributes, external,
5131 attributes->group, !!priv->fdb_def_rule,
5136 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5138 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5139 attributes->priority >= priority_max)
5140 return rte_flow_error_set(error, ENOTSUP,
5141 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5143 "priority out of range");
5144 if (attributes->transfer) {
5145 if (!priv->config.dv_esw_en)
5146 return rte_flow_error_set
5148 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5149 "E-Switch dr is not supported");
5150 if (!(priv->representor || priv->master))
5151 return rte_flow_error_set
5152 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5153 NULL, "E-Switch configuration can only be"
5154 " done by a master or a representor device");
5155 if (attributes->egress)
5156 return rte_flow_error_set
5158 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5159 "egress is not supported");
5161 if (!(attributes->egress ^ attributes->ingress))
5162 return rte_flow_error_set(error, ENOTSUP,
5163 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5164 "must specify exactly one of "
5165 "ingress or egress");
5170 * Internal validation function. For validating both actions and items.
5173 * Pointer to the rte_eth_dev structure.
5175 * Pointer to the flow attributes.
5177 * Pointer to the list of items.
5178 * @param[in] actions
5179 * Pointer to the list of actions.
5180 * @param[in] external
5181 * This flow rule is created by request external to PMD.
5182 * @param[in] hairpin
5183 * Number of hairpin TX actions, 0 means classic flow.
5185 * Pointer to the error structure.
5188 * 0 on success, a negative errno value otherwise and rte_errno is set.
5191 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5192 const struct rte_flow_item items[],
5193 const struct rte_flow_action actions[],
5194 bool external, int hairpin, struct rte_flow_error *error)
5197 uint64_t action_flags = 0;
5198 uint64_t item_flags = 0;
5199 uint64_t last_item = 0;
5200 uint8_t next_protocol = 0xff;
5201 uint16_t ether_type = 0;
5203 uint8_t item_ipv6_proto = 0;
5204 const struct rte_flow_item *gre_item = NULL;
5205 const struct rte_flow_action_raw_decap *decap;
5206 const struct rte_flow_action_raw_encap *encap;
5207 const struct rte_flow_action_rss *rss;
5208 const struct rte_flow_item_tcp nic_tcp_mask = {
5211 .src_port = RTE_BE16(UINT16_MAX),
5212 .dst_port = RTE_BE16(UINT16_MAX),
5215 const struct rte_flow_item_ipv4 nic_ipv4_mask = {
5217 .src_addr = RTE_BE32(0xffffffff),
5218 .dst_addr = RTE_BE32(0xffffffff),
5219 .type_of_service = 0xff,
5220 .next_proto_id = 0xff,
5221 .time_to_live = 0xff,
5224 const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5227 "\xff\xff\xff\xff\xff\xff\xff\xff"
5228 "\xff\xff\xff\xff\xff\xff\xff\xff",
5230 "\xff\xff\xff\xff\xff\xff\xff\xff"
5231 "\xff\xff\xff\xff\xff\xff\xff\xff",
5232 .vtc_flow = RTE_BE32(0xffffffff),
5237 const struct rte_flow_item_ecpri nic_ecpri_mask = {
5241 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5245 .dummy[0] = 0xffffffff,
5248 struct mlx5_priv *priv = dev->data->dev_private;
5249 struct mlx5_dev_config *dev_conf = &priv->config;
5250 uint16_t queue_index = 0xFFFF;
5251 const struct rte_flow_item_vlan *vlan_m = NULL;
5252 int16_t rw_act_num = 0;
5257 ret = flow_dv_validate_attributes(dev, attr, external, error);
5260 is_root = (uint64_t)ret;
5261 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5262 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5263 int type = items->type;
5265 if (!mlx5_flow_os_item_supported(type))
5266 return rte_flow_error_set(error, ENOTSUP,
5267 RTE_FLOW_ERROR_TYPE_ITEM,
5268 NULL, "item not supported");
5270 case RTE_FLOW_ITEM_TYPE_VOID:
5272 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5273 ret = flow_dv_validate_item_port_id
5274 (dev, items, attr, item_flags, error);
5277 last_item = MLX5_FLOW_ITEM_PORT_ID;
5279 case RTE_FLOW_ITEM_TYPE_ETH:
5280 ret = mlx5_flow_validate_item_eth(items, item_flags,
5284 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5285 MLX5_FLOW_LAYER_OUTER_L2;
5286 if (items->mask != NULL && items->spec != NULL) {
5288 ((const struct rte_flow_item_eth *)
5291 ((const struct rte_flow_item_eth *)
5293 ether_type = rte_be_to_cpu_16(ether_type);
5298 case RTE_FLOW_ITEM_TYPE_VLAN:
5299 ret = flow_dv_validate_item_vlan(items, item_flags,
5303 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5304 MLX5_FLOW_LAYER_OUTER_VLAN;
5305 if (items->mask != NULL && items->spec != NULL) {
5307 ((const struct rte_flow_item_vlan *)
5308 items->spec)->inner_type;
5310 ((const struct rte_flow_item_vlan *)
5311 items->mask)->inner_type;
5312 ether_type = rte_be_to_cpu_16(ether_type);
5316 /* Store outer VLAN mask for of_push_vlan action. */
5318 vlan_m = items->mask;
5320 case RTE_FLOW_ITEM_TYPE_IPV4:
5321 mlx5_flow_tunnel_ip_check(items, next_protocol,
5322 &item_flags, &tunnel);
5323 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
5330 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5331 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5332 if (items->mask != NULL &&
5333 ((const struct rte_flow_item_ipv4 *)
5334 items->mask)->hdr.next_proto_id) {
5336 ((const struct rte_flow_item_ipv4 *)
5337 (items->spec))->hdr.next_proto_id;
5339 ((const struct rte_flow_item_ipv4 *)
5340 (items->mask))->hdr.next_proto_id;
5342 /* Reset for inner layer. */
5343 next_protocol = 0xff;
5346 case RTE_FLOW_ITEM_TYPE_IPV6:
5347 mlx5_flow_tunnel_ip_check(items, next_protocol,
5348 &item_flags, &tunnel);
5349 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5356 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5357 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5358 if (items->mask != NULL &&
5359 ((const struct rte_flow_item_ipv6 *)
5360 items->mask)->hdr.proto) {
5362 ((const struct rte_flow_item_ipv6 *)
5363 items->spec)->hdr.proto;
5365 ((const struct rte_flow_item_ipv6 *)
5366 items->spec)->hdr.proto;
5368 ((const struct rte_flow_item_ipv6 *)
5369 items->mask)->hdr.proto;
5371 /* Reset for inner layer. */
5372 next_protocol = 0xff;
5375 case RTE_FLOW_ITEM_TYPE_TCP:
5376 ret = mlx5_flow_validate_item_tcp
5383 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5384 MLX5_FLOW_LAYER_OUTER_L4_TCP;
5386 case RTE_FLOW_ITEM_TYPE_UDP:
5387 ret = mlx5_flow_validate_item_udp(items, item_flags,
5392 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5393 MLX5_FLOW_LAYER_OUTER_L4_UDP;
5395 case RTE_FLOW_ITEM_TYPE_GRE:
5396 ret = mlx5_flow_validate_item_gre(items, item_flags,
5397 next_protocol, error);
5401 last_item = MLX5_FLOW_LAYER_GRE;
5403 case RTE_FLOW_ITEM_TYPE_NVGRE:
5404 ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5409 last_item = MLX5_FLOW_LAYER_NVGRE;
5411 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5412 ret = mlx5_flow_validate_item_gre_key
5413 (items, item_flags, gre_item, error);
5416 last_item = MLX5_FLOW_LAYER_GRE_KEY;
5418 case RTE_FLOW_ITEM_TYPE_VXLAN:
5419 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5423 last_item = MLX5_FLOW_LAYER_VXLAN;
5425 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5426 ret = mlx5_flow_validate_item_vxlan_gpe(items,
5431 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5433 case RTE_FLOW_ITEM_TYPE_GENEVE:
5434 ret = mlx5_flow_validate_item_geneve(items,
5439 last_item = MLX5_FLOW_LAYER_GENEVE;
5441 case RTE_FLOW_ITEM_TYPE_MPLS:
5442 ret = mlx5_flow_validate_item_mpls(dev, items,
5447 last_item = MLX5_FLOW_LAYER_MPLS;
5450 case RTE_FLOW_ITEM_TYPE_MARK:
5451 ret = flow_dv_validate_item_mark(dev, items, attr,
5455 last_item = MLX5_FLOW_ITEM_MARK;
5457 case RTE_FLOW_ITEM_TYPE_META:
5458 ret = flow_dv_validate_item_meta(dev, items, attr,
5462 last_item = MLX5_FLOW_ITEM_METADATA;
5464 case RTE_FLOW_ITEM_TYPE_ICMP:
5465 ret = mlx5_flow_validate_item_icmp(items, item_flags,
5470 last_item = MLX5_FLOW_LAYER_ICMP;
5472 case RTE_FLOW_ITEM_TYPE_ICMP6:
5473 ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5478 item_ipv6_proto = IPPROTO_ICMPV6;
5479 last_item = MLX5_FLOW_LAYER_ICMP6;
5481 case RTE_FLOW_ITEM_TYPE_TAG:
5482 ret = flow_dv_validate_item_tag(dev, items,
5486 last_item = MLX5_FLOW_ITEM_TAG;
5488 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5489 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5491 case RTE_FLOW_ITEM_TYPE_GTP:
5492 ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5496 last_item = MLX5_FLOW_LAYER_GTP;
5498 case RTE_FLOW_ITEM_TYPE_ECPRI:
5499 /* Capacity will be checked in the translate stage. */
5500 ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5507 last_item = MLX5_FLOW_LAYER_ECPRI;
5510 return rte_flow_error_set(error, ENOTSUP,
5511 RTE_FLOW_ERROR_TYPE_ITEM,
5512 NULL, "item not supported");
5514 item_flags |= last_item;
5516 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5517 int type = actions->type;
5519 if (!mlx5_flow_os_action_supported(type))
5520 return rte_flow_error_set(error, ENOTSUP,
5521 RTE_FLOW_ERROR_TYPE_ACTION,
5523 "action not supported");
5524 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5525 return rte_flow_error_set(error, ENOTSUP,
5526 RTE_FLOW_ERROR_TYPE_ACTION,
5527 actions, "too many actions");
5529 case RTE_FLOW_ACTION_TYPE_VOID:
5531 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5532 ret = flow_dv_validate_action_port_id(dev,
5539 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5542 case RTE_FLOW_ACTION_TYPE_FLAG:
5543 ret = flow_dv_validate_action_flag(dev, action_flags,
5547 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5548 /* Count all modify-header actions as one. */
5549 if (!(action_flags &
5550 MLX5_FLOW_MODIFY_HDR_ACTIONS))
5552 action_flags |= MLX5_FLOW_ACTION_FLAG |
5553 MLX5_FLOW_ACTION_MARK_EXT;
5555 action_flags |= MLX5_FLOW_ACTION_FLAG;
5558 rw_act_num += MLX5_ACT_NUM_SET_MARK;
5560 case RTE_FLOW_ACTION_TYPE_MARK:
5561 ret = flow_dv_validate_action_mark(dev, actions,
5566 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5567 /* Count all modify-header actions as one. */
5568 if (!(action_flags &
5569 MLX5_FLOW_MODIFY_HDR_ACTIONS))
5571 action_flags |= MLX5_FLOW_ACTION_MARK |
5572 MLX5_FLOW_ACTION_MARK_EXT;
5574 action_flags |= MLX5_FLOW_ACTION_MARK;
5577 rw_act_num += MLX5_ACT_NUM_SET_MARK;
5579 case RTE_FLOW_ACTION_TYPE_SET_META:
5580 ret = flow_dv_validate_action_set_meta(dev, actions,
5585 /* Count all modify-header actions as one action. */
5586 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5588 action_flags |= MLX5_FLOW_ACTION_SET_META;
5589 rw_act_num += MLX5_ACT_NUM_SET_META;
5591 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5592 ret = flow_dv_validate_action_set_tag(dev, actions,
5597 /* Count all modify-header actions as one action. */
5598 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5600 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5601 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5603 case RTE_FLOW_ACTION_TYPE_DROP:
5604 ret = mlx5_flow_validate_action_drop(action_flags,
5608 action_flags |= MLX5_FLOW_ACTION_DROP;
5611 case RTE_FLOW_ACTION_TYPE_QUEUE:
5612 ret = mlx5_flow_validate_action_queue(actions,
5617 queue_index = ((const struct rte_flow_action_queue *)
5618 (actions->conf))->index;
5619 action_flags |= MLX5_FLOW_ACTION_QUEUE;
5622 case RTE_FLOW_ACTION_TYPE_RSS:
5623 rss = actions->conf;
5624 ret = mlx5_flow_validate_action_rss(actions,
5630 if (rss != NULL && rss->queue_num)
5631 queue_index = rss->queue[0];
5632 action_flags |= MLX5_FLOW_ACTION_RSS;
5635 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5637 mlx5_flow_validate_action_default_miss(action_flags,
5641 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5644 case RTE_FLOW_ACTION_TYPE_COUNT:
5645 ret = flow_dv_validate_action_count(dev, error);
5648 action_flags |= MLX5_FLOW_ACTION_COUNT;
5651 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5652 if (flow_dv_validate_action_pop_vlan(dev,
5658 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5661 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5662 ret = flow_dv_validate_action_push_vlan(dev,
5669 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5672 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5673 ret = flow_dv_validate_action_set_vlan_pcp
5674 (action_flags, actions, error);
5677 /* Count PCP with push_vlan command. */
5678 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5680 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5681 ret = flow_dv_validate_action_set_vlan_vid
5682 (item_flags, action_flags,
5686 /* Count VID with push_vlan command. */
5687 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5688 rw_act_num += MLX5_ACT_NUM_MDF_VID;
5690 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5691 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5692 ret = flow_dv_validate_action_l2_encap(dev,
5698 action_flags |= MLX5_FLOW_ACTION_ENCAP;
5701 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5702 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5703 ret = flow_dv_validate_action_decap(dev, action_flags,
5707 action_flags |= MLX5_FLOW_ACTION_DECAP;
5710 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5711 ret = flow_dv_validate_action_raw_encap_decap
5712 (dev, NULL, actions->conf, attr, &action_flags,
5717 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5718 decap = actions->conf;
5719 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5721 if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5725 encap = actions->conf;
5727 ret = flow_dv_validate_action_raw_encap_decap
5729 decap ? decap : &empty_decap, encap,
5730 attr, &action_flags, &actions_n,
5735 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5736 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5737 ret = flow_dv_validate_action_modify_mac(action_flags,
5743 /* Count all modify-header actions as one action. */
5744 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5746 action_flags |= actions->type ==
5747 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5748 MLX5_FLOW_ACTION_SET_MAC_SRC :
5749 MLX5_FLOW_ACTION_SET_MAC_DST;
5751 * Even if the source and destination MAC addresses have
5752 * overlap in the header with 4B alignment, the convert
5753 * function will handle them separately and 4 SW actions
5754 * will be created. And 2 actions will be added each
5755 * time no matter how many bytes of address will be set.
5757 rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5759 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5760 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5761 ret = flow_dv_validate_action_modify_ipv4(action_flags,
5767 /* Count all modify-header actions as one action. */
5768 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5770 action_flags |= actions->type ==
5771 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5772 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5773 MLX5_FLOW_ACTION_SET_IPV4_DST;
5774 rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5776 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5777 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5778 ret = flow_dv_validate_action_modify_ipv6(action_flags,
5784 if (item_ipv6_proto == IPPROTO_ICMPV6)
5785 return rte_flow_error_set(error, ENOTSUP,
5786 RTE_FLOW_ERROR_TYPE_ACTION,
5788 "Can't change header "
5789 "with ICMPv6 proto");
5790 /* Count all modify-header actions as one action. */
5791 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5793 action_flags |= actions->type ==
5794 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5795 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5796 MLX5_FLOW_ACTION_SET_IPV6_DST;
5797 rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5799 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5800 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5801 ret = flow_dv_validate_action_modify_tp(action_flags,
5807 /* Count all modify-header actions as one action. */
5808 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5810 action_flags |= actions->type ==
5811 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5812 MLX5_FLOW_ACTION_SET_TP_SRC :
5813 MLX5_FLOW_ACTION_SET_TP_DST;
5814 rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5816 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5817 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5818 ret = flow_dv_validate_action_modify_ttl(action_flags,
5824 /* Count all modify-header actions as one action. */
5825 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5827 action_flags |= actions->type ==
5828 RTE_FLOW_ACTION_TYPE_SET_TTL ?
5829 MLX5_FLOW_ACTION_SET_TTL :
5830 MLX5_FLOW_ACTION_DEC_TTL;
5831 rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5833 case RTE_FLOW_ACTION_TYPE_JUMP:
5834 ret = flow_dv_validate_action_jump(actions,
5841 action_flags |= MLX5_FLOW_ACTION_JUMP;
5843 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5844 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5845 ret = flow_dv_validate_action_modify_tcp_seq
5852 /* Count all modify-header actions as one action. */
5853 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5855 action_flags |= actions->type ==
5856 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5857 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5858 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5859 rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5861 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5862 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5863 ret = flow_dv_validate_action_modify_tcp_ack
5870 /* Count all modify-header actions as one action. */
5871 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5873 action_flags |= actions->type ==
5874 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5875 MLX5_FLOW_ACTION_INC_TCP_ACK :
5876 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5877 rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5879 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5881 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5882 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5883 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5885 case RTE_FLOW_ACTION_TYPE_METER:
5886 ret = mlx5_flow_validate_action_meter(dev,
5892 action_flags |= MLX5_FLOW_ACTION_METER;
5894 /* Meter action will add one more TAG action. */
5895 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5897 case RTE_FLOW_ACTION_TYPE_AGE:
5898 ret = flow_dv_validate_action_age(action_flags,
5903 action_flags |= MLX5_FLOW_ACTION_AGE;
5906 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5907 ret = flow_dv_validate_action_modify_ipv4_dscp
5914 /* Count all modify-header actions as one action. */
5915 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5917 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5918 rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5920 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5921 ret = flow_dv_validate_action_modify_ipv6_dscp
5928 /* Count all modify-header actions as one action. */
5929 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5931 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5932 rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5934 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5935 ret = flow_dv_validate_action_sample(action_flags,
5940 action_flags |= MLX5_FLOW_ACTION_SAMPLE;
5944 return rte_flow_error_set(error, ENOTSUP,
5945 RTE_FLOW_ERROR_TYPE_ACTION,
5947 "action not supported");
5951 * Validate the drop action mutual exclusion with other actions.
5952 * Drop action is mutually-exclusive with any other action, except for
5955 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5956 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5957 return rte_flow_error_set(error, EINVAL,
5958 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5959 "Drop action is mutually-exclusive "
5960 "with any other action, except for "
5962 /* Eswitch has few restrictions on using items and actions */
5963 if (attr->transfer) {
5964 if (!mlx5_flow_ext_mreg_supported(dev) &&
5965 action_flags & MLX5_FLOW_ACTION_FLAG)
5966 return rte_flow_error_set(error, ENOTSUP,
5967 RTE_FLOW_ERROR_TYPE_ACTION,
5969 "unsupported action FLAG");
5970 if (!mlx5_flow_ext_mreg_supported(dev) &&
5971 action_flags & MLX5_FLOW_ACTION_MARK)
5972 return rte_flow_error_set(error, ENOTSUP,
5973 RTE_FLOW_ERROR_TYPE_ACTION,
5975 "unsupported action MARK");
5976 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5977 return rte_flow_error_set(error, ENOTSUP,
5978 RTE_FLOW_ERROR_TYPE_ACTION,
5980 "unsupported action QUEUE");
5981 if (action_flags & MLX5_FLOW_ACTION_RSS)
5982 return rte_flow_error_set(error, ENOTSUP,
5983 RTE_FLOW_ERROR_TYPE_ACTION,
5985 "unsupported action RSS");
5986 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5987 return rte_flow_error_set(error, EINVAL,
5988 RTE_FLOW_ERROR_TYPE_ACTION,
5990 "no fate action is found");
5992 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5993 return rte_flow_error_set(error, EINVAL,
5994 RTE_FLOW_ERROR_TYPE_ACTION,
5996 "no fate action is found");
5998 /* Continue validation for Xcap and VLAN actions.*/
5999 if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6000 MLX5_FLOW_VLAN_ACTIONS)) &&
6001 (queue_index == 0xFFFF ||
6002 mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6003 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6004 MLX5_FLOW_XCAP_ACTIONS)
6005 return rte_flow_error_set(error, ENOTSUP,
6006 RTE_FLOW_ERROR_TYPE_ACTION,
6007 NULL, "encap and decap "
6008 "combination aren't supported");
6009 if (!attr->transfer && attr->ingress) {
6010 if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6011 return rte_flow_error_set
6013 RTE_FLOW_ERROR_TYPE_ACTION,
6014 NULL, "encap is not supported"
6015 " for ingress traffic");
6016 else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6017 return rte_flow_error_set
6019 RTE_FLOW_ERROR_TYPE_ACTION,
6020 NULL, "push VLAN action not "
6021 "supported for ingress");
6022 else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6023 MLX5_FLOW_VLAN_ACTIONS)
6024 return rte_flow_error_set
6026 RTE_FLOW_ERROR_TYPE_ACTION,
6027 NULL, "no support for "
6028 "multiple VLAN actions");
6031 /* Hairpin flow will add one more TAG action. */
6033 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6034 /* extra metadata enabled: one more TAG action will be add. */
6035 if (dev_conf->dv_flow_en &&
6036 dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6037 mlx5_flow_ext_mreg_supported(dev))
6038 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6039 if ((uint32_t)rw_act_num >
6040 flow_dv_modify_hdr_action_max(dev, is_root)) {
6041 return rte_flow_error_set(error, ENOTSUP,
6042 RTE_FLOW_ERROR_TYPE_ACTION,
6043 NULL, "too many header modify"
6044 " actions to support");
6050 * Internal preparation function. Allocates the DV flow size,
6051 * this size is constant.
6054 * Pointer to the rte_eth_dev structure.
6056 * Pointer to the flow attributes.
6058 * Pointer to the list of items.
6059 * @param[in] actions
6060 * Pointer to the list of actions.
6062 * Pointer to the error structure.
6065 * Pointer to mlx5_flow object on success,
6066 * otherwise NULL and rte_errno is set.
6068 static struct mlx5_flow *
6069 flow_dv_prepare(struct rte_eth_dev *dev,
6070 const struct rte_flow_attr *attr __rte_unused,
6071 const struct rte_flow_item items[] __rte_unused,
6072 const struct rte_flow_action actions[] __rte_unused,
6073 struct rte_flow_error *error)
6075 uint32_t handle_idx = 0;
6076 struct mlx5_flow *dev_flow;
6077 struct mlx5_flow_handle *dev_handle;
6078 struct mlx5_priv *priv = dev->data->dev_private;
6080 /* In case of corrupting the memory. */
6081 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6082 rte_flow_error_set(error, ENOSPC,
6083 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6084 "not free temporary device flow");
6087 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6090 rte_flow_error_set(error, ENOMEM,
6091 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6092 "not enough memory to create flow handle");
6095 /* No multi-thread supporting. */
6096 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6097 dev_flow->handle = dev_handle;
6098 dev_flow->handle_idx = handle_idx;
6100 * In some old rdma-core releases, before continuing, a check of the
6101 * length of matching parameter will be done at first. It needs to use
6102 * the length without misc4 param. If the flow has misc4 support, then
6103 * the length needs to be adjusted accordingly. Each param member is
6104 * aligned with a 64B boundary naturally.
6106 dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6107 MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6109 * The matching value needs to be cleared to 0 before using. In the
6110 * past, it will be automatically cleared when using rte_*alloc
6111 * API. The time consumption will be almost the same as before.
6113 memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6114 dev_flow->ingress = attr->ingress;
6115 dev_flow->dv.transfer = attr->transfer;
6119 #ifdef RTE_LIBRTE_MLX5_DEBUG
6121 * Sanity check for match mask and value. Similar to check_valid_spec() in
6122 * kernel driver. If unmasked bit is present in value, it returns failure.
6125 * pointer to match mask buffer.
6126 * @param match_value
6127 * pointer to match value buffer.
6130 * 0 if valid, -EINVAL otherwise.
6133 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6135 uint8_t *m = match_mask;
6136 uint8_t *v = match_value;
6139 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6142 "match_value differs from match_criteria"
6143 " %p[%u] != %p[%u]",
6144 match_value, i, match_mask, i);
6153 * Add match of ip_version.
6157 * @param[in] headers_v
6158 * Values header pointer.
6159 * @param[in] headers_m
6160 * Masks header pointer.
6161 * @param[in] ip_version
6162 * The IP version to set.
6165 flow_dv_set_match_ip_version(uint32_t group,
6171 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6173 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6175 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6176 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6177 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6181 * Add Ethernet item to matcher and to the value.
6183 * @param[in, out] matcher
6185 * @param[in, out] key
6186 * Flow matcher value.
6188 * Flow pattern to translate.
6190 * Item is inner pattern.
6193 flow_dv_translate_item_eth(void *matcher, void *key,
6194 const struct rte_flow_item *item, int inner,
6197 const struct rte_flow_item_eth *eth_m = item->mask;
6198 const struct rte_flow_item_eth *eth_v = item->spec;
6199 const struct rte_flow_item_eth nic_mask = {
6200 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6201 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6202 .type = RTE_BE16(0xffff),
6214 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6216 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6218 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6220 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6222 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6223 ð_m->dst, sizeof(eth_m->dst));
6224 /* The value must be in the range of the mask. */
6225 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6226 for (i = 0; i < sizeof(eth_m->dst); ++i)
6227 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6228 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6229 ð_m->src, sizeof(eth_m->src));
6230 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6231 /* The value must be in the range of the mask. */
6232 for (i = 0; i < sizeof(eth_m->dst); ++i)
6233 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6235 /* When ethertype is present set mask for tagged VLAN. */
6236 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6237 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6238 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6239 eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6240 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6242 /* Return here to avoid setting match on ethertype. */
6247 * HW supports match on one Ethertype, the Ethertype following the last
6248 * VLAN tag of the packet (see PRM).
6249 * Set match on ethertype only if ETH header is not followed by VLAN.
6250 * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6251 * ethertype, and use ip_version field instead.
6252 * eCPRI over Ether layer will use type value 0xAEFE.
6254 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6255 eth_m->type == 0xFFFF) {
6256 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6257 } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6258 eth_m->type == 0xFFFF) {
6259 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6261 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6262 rte_be_to_cpu_16(eth_m->type));
6263 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6265 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6270 * Add VLAN item to matcher and to the value.
6272 * @param[in, out] dev_flow
6274 * @param[in, out] matcher
6276 * @param[in, out] key
6277 * Flow matcher value.
6279 * Flow pattern to translate.
6281 * Item is inner pattern.
6284 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6285 void *matcher, void *key,
6286 const struct rte_flow_item *item,
6287 int inner, uint32_t group)
6289 const struct rte_flow_item_vlan *vlan_m = item->mask;
6290 const struct rte_flow_item_vlan *vlan_v = item->spec;
6297 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6299 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6301 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6303 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6305 * This is workaround, masks are not supported,
6306 * and pre-validated.
6309 dev_flow->handle->vf_vlan.tag =
6310 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6313 * When VLAN item exists in flow, mark packet as tagged,
6314 * even if TCI is not specified.
6316 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6317 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6321 vlan_m = &rte_flow_item_vlan_mask;
6322 tci_m = rte_be_to_cpu_16(vlan_m->tci);
6323 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6324 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6325 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6326 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6327 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6328 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6329 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6331 * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6332 * ethertype, and use ip_version field instead.
6334 if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6335 vlan_m->inner_type == 0xFFFF) {
6336 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6337 } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6338 vlan_m->inner_type == 0xFFFF) {
6339 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6341 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6342 rte_be_to_cpu_16(vlan_m->inner_type));
6343 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6344 rte_be_to_cpu_16(vlan_m->inner_type &
6345 vlan_v->inner_type));
6350 * Add IPV4 item to matcher and to the value.
6352 * @param[in, out] matcher
6354 * @param[in, out] key
6355 * Flow matcher value.
6357 * Flow pattern to translate.
6358 * @param[in] item_flags
6359 * Bit-fields that holds the items detected until now.
6361 * Item is inner pattern.
6363 * The group to insert the rule.
6366 flow_dv_translate_item_ipv4(void *matcher, void *key,
6367 const struct rte_flow_item *item,
6368 const uint64_t item_flags,
6369 int inner, uint32_t group)
6371 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6372 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6373 const struct rte_flow_item_ipv4 nic_mask = {
6375 .src_addr = RTE_BE32(0xffffffff),
6376 .dst_addr = RTE_BE32(0xffffffff),
6377 .type_of_service = 0xff,
6378 .next_proto_id = 0xff,
6379 .time_to_live = 0xff,
6389 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6391 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6393 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6395 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6397 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6399 * On outer header (which must contains L2), or inner header with L2,
6400 * set cvlan_tag mask bit to mark this packet as untagged.
6401 * This should be done even if item->spec is empty.
6403 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6404 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6409 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6410 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6411 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6412 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6413 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6414 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6415 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6416 src_ipv4_src_ipv6.ipv4_layout.ipv4);
6417 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6418 src_ipv4_src_ipv6.ipv4_layout.ipv4);
6419 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6420 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6421 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6422 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6423 ipv4_m->hdr.type_of_service);
6424 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6425 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6426 ipv4_m->hdr.type_of_service >> 2);
6427 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6428 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6429 ipv4_m->hdr.next_proto_id);
6430 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6431 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6432 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6433 ipv4_m->hdr.time_to_live);
6434 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6435 ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6439 * Add IPV6 item to matcher and to the value.
6441 * @param[in, out] matcher
6443 * @param[in, out] key
6444 * Flow matcher value.
6446 * Flow pattern to translate.
6447 * @param[in] item_flags
6448 * Bit-fields that holds the items detected until now.
6450 * Item is inner pattern.
6452 * The group to insert the rule.
6455 flow_dv_translate_item_ipv6(void *matcher, void *key,
6456 const struct rte_flow_item *item,
6457 const uint64_t item_flags,
6458 int inner, uint32_t group)
6460 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6461 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6462 const struct rte_flow_item_ipv6 nic_mask = {
6465 "\xff\xff\xff\xff\xff\xff\xff\xff"
6466 "\xff\xff\xff\xff\xff\xff\xff\xff",
6468 "\xff\xff\xff\xff\xff\xff\xff\xff"
6469 "\xff\xff\xff\xff\xff\xff\xff\xff",
6470 .vtc_flow = RTE_BE32(0xffffffff),
6477 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6478 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6487 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6489 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6491 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6493 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6495 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6497 * On outer header (which must contains L2), or inner header with L2,
6498 * set cvlan_tag mask bit to mark this packet as untagged.
6499 * This should be done even if item->spec is empty.
6501 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6502 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6507 size = sizeof(ipv6_m->hdr.dst_addr);
6508 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6509 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6510 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6511 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6512 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6513 for (i = 0; i < size; ++i)
6514 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6515 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6516 src_ipv4_src_ipv6.ipv6_layout.ipv6);
6517 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6518 src_ipv4_src_ipv6.ipv6_layout.ipv6);
6519 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6520 for (i = 0; i < size; ++i)
6521 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6523 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6524 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6525 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6526 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6527 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6528 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6531 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6533 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6536 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6538 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6542 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6544 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6545 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6547 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6548 ipv6_m->hdr.hop_limits);
6549 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6550 ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6554 * Add TCP item to matcher and to the value.
6556 * @param[in, out] matcher
6558 * @param[in, out] key
6559 * Flow matcher value.
6561 * Flow pattern to translate.
6563 * Item is inner pattern.
6566 flow_dv_translate_item_tcp(void *matcher, void *key,
6567 const struct rte_flow_item *item,
6570 const struct rte_flow_item_tcp *tcp_m = item->mask;
6571 const struct rte_flow_item_tcp *tcp_v = item->spec;
6576 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6578 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6580 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6582 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6584 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6585 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6589 tcp_m = &rte_flow_item_tcp_mask;
6590 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6591 rte_be_to_cpu_16(tcp_m->hdr.src_port));
6592 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6593 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6594 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6595 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6596 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6597 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6598 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6599 tcp_m->hdr.tcp_flags);
6600 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6601 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6605 * Add UDP item to matcher and to the value.
6607 * @param[in, out] matcher
6609 * @param[in, out] key
6610 * Flow matcher value.
6612 * Flow pattern to translate.
6614 * Item is inner pattern.
6617 flow_dv_translate_item_udp(void *matcher, void *key,
6618 const struct rte_flow_item *item,
6621 const struct rte_flow_item_udp *udp_m = item->mask;
6622 const struct rte_flow_item_udp *udp_v = item->spec;
6627 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6629 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6631 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6633 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6635 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6636 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6640 udp_m = &rte_flow_item_udp_mask;
6641 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6642 rte_be_to_cpu_16(udp_m->hdr.src_port));
6643 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6644 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6645 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6646 rte_be_to_cpu_16(udp_m->hdr.dst_port));
6647 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6648 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6652 * Add GRE optional Key item to matcher and to the value.
6654 * @param[in, out] matcher
6656 * @param[in, out] key
6657 * Flow matcher value.
6659 * Flow pattern to translate.
6661 * Item is inner pattern.
6664 flow_dv_translate_item_gre_key(void *matcher, void *key,
6665 const struct rte_flow_item *item)
6667 const rte_be32_t *key_m = item->mask;
6668 const rte_be32_t *key_v = item->spec;
6669 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6670 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6671 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6673 /* GRE K bit must be on and should already be validated */
6674 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6675 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6679 key_m = &gre_key_default_mask;
6680 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6681 rte_be_to_cpu_32(*key_m) >> 8);
6682 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6683 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6684 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6685 rte_be_to_cpu_32(*key_m) & 0xFF);
6686 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6687 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6691 * Add GRE item to matcher and to the value.
6693 * @param[in, out] matcher
6695 * @param[in, out] key
6696 * Flow matcher value.
6698 * Flow pattern to translate.
6700 * Item is inner pattern.
6703 flow_dv_translate_item_gre(void *matcher, void *key,
6704 const struct rte_flow_item *item,
6707 const struct rte_flow_item_gre *gre_m = item->mask;
6708 const struct rte_flow_item_gre *gre_v = item->spec;
6711 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6712 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6719 uint16_t s_present:1;
6720 uint16_t k_present:1;
6721 uint16_t rsvd_bit1:1;
6722 uint16_t c_present:1;
6726 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6729 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6731 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6733 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6735 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6737 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6738 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6742 gre_m = &rte_flow_item_gre_mask;
6743 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6744 rte_be_to_cpu_16(gre_m->protocol));
6745 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6746 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6747 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6748 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6749 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6750 gre_crks_rsvd0_ver_m.c_present);
6751 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6752 gre_crks_rsvd0_ver_v.c_present &
6753 gre_crks_rsvd0_ver_m.c_present);
6754 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6755 gre_crks_rsvd0_ver_m.k_present);
6756 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6757 gre_crks_rsvd0_ver_v.k_present &
6758 gre_crks_rsvd0_ver_m.k_present);
6759 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6760 gre_crks_rsvd0_ver_m.s_present);
6761 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6762 gre_crks_rsvd0_ver_v.s_present &
6763 gre_crks_rsvd0_ver_m.s_present);
6767 * Add NVGRE item to matcher and to the value.
6769 * @param[in, out] matcher
6771 * @param[in, out] key
6772 * Flow matcher value.
6774 * Flow pattern to translate.
6776 * Item is inner pattern.
6779 flow_dv_translate_item_nvgre(void *matcher, void *key,
6780 const struct rte_flow_item *item,
6783 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6784 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6785 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6786 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6787 const char *tni_flow_id_m;
6788 const char *tni_flow_id_v;
6794 /* For NVGRE, GRE header fields must be set with defined values. */
6795 const struct rte_flow_item_gre gre_spec = {
6796 .c_rsvd0_ver = RTE_BE16(0x2000),
6797 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6799 const struct rte_flow_item_gre gre_mask = {
6800 .c_rsvd0_ver = RTE_BE16(0xB000),
6801 .protocol = RTE_BE16(UINT16_MAX),
6803 const struct rte_flow_item gre_item = {
6808 flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6812 nvgre_m = &rte_flow_item_nvgre_mask;
6813 tni_flow_id_m = (const char *)nvgre_m->tni;
6814 tni_flow_id_v = (const char *)nvgre_v->tni;
6815 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6816 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6817 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6818 memcpy(gre_key_m, tni_flow_id_m, size);
6819 for (i = 0; i < size; ++i)
6820 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6824 * Add VXLAN item to matcher and to the value.
6826 * @param[in, out] matcher
6828 * @param[in, out] key
6829 * Flow matcher value.
6831 * Flow pattern to translate.
6833 * Item is inner pattern.
6836 flow_dv_translate_item_vxlan(void *matcher, void *key,
6837 const struct rte_flow_item *item,
6840 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6841 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6844 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6845 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6853 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6855 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6857 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6859 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6861 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6862 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6863 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6864 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6865 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6870 vxlan_m = &rte_flow_item_vxlan_mask;
6871 size = sizeof(vxlan_m->vni);
6872 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6873 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6874 memcpy(vni_m, vxlan_m->vni, size);
6875 for (i = 0; i < size; ++i)
6876 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6880 * Add VXLAN-GPE item to matcher and to the value.
6882 * @param[in, out] matcher
6884 * @param[in, out] key
6885 * Flow matcher value.
6887 * Flow pattern to translate.
6889 * Item is inner pattern.
6893 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6894 const struct rte_flow_item *item, int inner)
6896 const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6897 const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6901 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6903 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6909 uint8_t flags_m = 0xff;
6910 uint8_t flags_v = 0xc;
6913 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6915 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6917 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6919 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6921 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6922 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6923 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6924 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6925 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6930 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6931 size = sizeof(vxlan_m->vni);
6932 vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6933 vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6934 memcpy(vni_m, vxlan_m->vni, size);
6935 for (i = 0; i < size; ++i)
6936 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6937 if (vxlan_m->flags) {
6938 flags_m = vxlan_m->flags;
6939 flags_v = vxlan_v->flags;
6941 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6942 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6943 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6945 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6950 * Add Geneve item to matcher and to the value.
6952 * @param[in, out] matcher
6954 * @param[in, out] key
6955 * Flow matcher value.
6957 * Flow pattern to translate.
6959 * Item is inner pattern.
6963 flow_dv_translate_item_geneve(void *matcher, void *key,
6964 const struct rte_flow_item *item, int inner)
6966 const struct rte_flow_item_geneve *geneve_m = item->mask;
6967 const struct rte_flow_item_geneve *geneve_v = item->spec;
6970 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6971 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6980 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6982 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6984 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6986 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6988 dport = MLX5_UDP_PORT_GENEVE;
6989 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6990 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6991 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6996 geneve_m = &rte_flow_item_geneve_mask;
6997 size = sizeof(geneve_m->vni);
6998 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6999 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7000 memcpy(vni_m, geneve_m->vni, size);
7001 for (i = 0; i < size; ++i)
7002 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7003 MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7004 rte_be_to_cpu_16(geneve_m->protocol));
7005 MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7006 rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7007 gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7008 gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7009 MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7010 MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7011 MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7012 MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7013 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7014 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7015 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7016 MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7017 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7021 * Add MPLS item to matcher and to the value.
7023 * @param[in, out] matcher
7025 * @param[in, out] key
7026 * Flow matcher value.
7028 * Flow pattern to translate.
7029 * @param[in] prev_layer
7030 * The protocol layer indicated in previous item.
7032 * Item is inner pattern.
7035 flow_dv_translate_item_mpls(void *matcher, void *key,
7036 const struct rte_flow_item *item,
7037 uint64_t prev_layer,
7040 const uint32_t *in_mpls_m = item->mask;
7041 const uint32_t *in_mpls_v = item->spec;
7042 uint32_t *out_mpls_m = 0;
7043 uint32_t *out_mpls_v = 0;
7044 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7045 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7046 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7048 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7049 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7050 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7052 switch (prev_layer) {
7053 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7054 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7055 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7056 MLX5_UDP_PORT_MPLS);
7058 case MLX5_FLOW_LAYER_GRE:
7059 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7060 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7061 RTE_ETHER_TYPE_MPLS);
7064 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7065 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7072 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7073 switch (prev_layer) {
7074 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7076 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7077 outer_first_mpls_over_udp);
7079 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7080 outer_first_mpls_over_udp);
7082 case MLX5_FLOW_LAYER_GRE:
7084 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7085 outer_first_mpls_over_gre);
7087 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7088 outer_first_mpls_over_gre);
7091 /* Inner MPLS not over GRE is not supported. */
7094 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7098 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7104 if (out_mpls_m && out_mpls_v) {
7105 *out_mpls_m = *in_mpls_m;
7106 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7111 * Add metadata register item to matcher
7113 * @param[in, out] matcher
7115 * @param[in, out] key
7116 * Flow matcher value.
7117 * @param[in] reg_type
7118 * Type of device metadata register
7125 flow_dv_match_meta_reg(void *matcher, void *key,
7126 enum modify_reg reg_type,
7127 uint32_t data, uint32_t mask)
7130 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7132 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7138 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7139 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7142 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7143 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7147 * The metadata register C0 field might be divided into
7148 * source vport index and META item value, we should set
7149 * this field according to specified mask, not as whole one.
7151 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7153 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7154 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7157 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7160 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7161 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7164 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7165 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7168 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7169 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7172 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7173 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7176 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7177 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7180 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7181 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7184 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7185 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7194 * Add MARK item to matcher
7197 * The device to configure through.
7198 * @param[in, out] matcher
7200 * @param[in, out] key
7201 * Flow matcher value.
7203 * Flow pattern to translate.
7206 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7207 void *matcher, void *key,
7208 const struct rte_flow_item *item)
7210 struct mlx5_priv *priv = dev->data->dev_private;
7211 const struct rte_flow_item_mark *mark;
7215 mark = item->mask ? (const void *)item->mask :
7216 &rte_flow_item_mark_mask;
7217 mask = mark->id & priv->sh->dv_mark_mask;
7218 mark = (const void *)item->spec;
7220 value = mark->id & priv->sh->dv_mark_mask & mask;
7222 enum modify_reg reg;
7224 /* Get the metadata register index for the mark. */
7225 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7226 MLX5_ASSERT(reg > 0);
7227 if (reg == REG_C_0) {
7228 struct mlx5_priv *priv = dev->data->dev_private;
7229 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7230 uint32_t shl_c0 = rte_bsf32(msk_c0);
7236 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7241 * Add META item to matcher
7244 * The devich to configure through.
7245 * @param[in, out] matcher
7247 * @param[in, out] key
7248 * Flow matcher value.
7250 * Attributes of flow that includes this item.
7252 * Flow pattern to translate.
7255 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7256 void *matcher, void *key,
7257 const struct rte_flow_attr *attr,
7258 const struct rte_flow_item *item)
7260 const struct rte_flow_item_meta *meta_m;
7261 const struct rte_flow_item_meta *meta_v;
7263 meta_m = (const void *)item->mask;
7265 meta_m = &rte_flow_item_meta_mask;
7266 meta_v = (const void *)item->spec;
7269 uint32_t value = meta_v->data;
7270 uint32_t mask = meta_m->data;
7272 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7276 * In datapath code there is no endianness
7277 * coversions for perfromance reasons, all
7278 * pattern conversions are done in rte_flow.
7280 value = rte_cpu_to_be_32(value);
7281 mask = rte_cpu_to_be_32(mask);
7282 if (reg == REG_C_0) {
7283 struct mlx5_priv *priv = dev->data->dev_private;
7284 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7285 uint32_t shl_c0 = rte_bsf32(msk_c0);
7286 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7287 uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7294 MLX5_ASSERT(msk_c0);
7295 MLX5_ASSERT(!(~msk_c0 & mask));
7297 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7302 * Add vport metadata Reg C0 item to matcher
7304 * @param[in, out] matcher
7306 * @param[in, out] key
7307 * Flow matcher value.
7309 * Flow pattern to translate.
7312 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7313 uint32_t value, uint32_t mask)
7315 flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7319 * Add tag item to matcher
7322 * The devich to configure through.
7323 * @param[in, out] matcher
7325 * @param[in, out] key
7326 * Flow matcher value.
7328 * Flow pattern to translate.
7331 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7332 void *matcher, void *key,
7333 const struct rte_flow_item *item)
7335 const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7336 const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7337 uint32_t mask, value;
7340 value = tag_v->data;
7341 mask = tag_m ? tag_m->data : UINT32_MAX;
7342 if (tag_v->id == REG_C_0) {
7343 struct mlx5_priv *priv = dev->data->dev_private;
7344 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7345 uint32_t shl_c0 = rte_bsf32(msk_c0);
7351 flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7355 * Add TAG item to matcher
7358 * The devich to configure through.
7359 * @param[in, out] matcher
7361 * @param[in, out] key
7362 * Flow matcher value.
7364 * Flow pattern to translate.
7367 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7368 void *matcher, void *key,
7369 const struct rte_flow_item *item)
7371 const struct rte_flow_item_tag *tag_v = item->spec;
7372 const struct rte_flow_item_tag *tag_m = item->mask;
7373 enum modify_reg reg;
7376 tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7377 /* Get the metadata register index for the tag. */
7378 reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7379 MLX5_ASSERT(reg > 0);
7380 flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7384 * Add source vport match to the specified matcher.
7386 * @param[in, out] matcher
7388 * @param[in, out] key
7389 * Flow matcher value.
7391 * Source vport value to match
7396 flow_dv_translate_item_source_vport(void *matcher, void *key,
7397 int16_t port, uint16_t mask)
7399 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7400 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7402 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7403 MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7407 * Translate port-id item to eswitch match on port-id.
7410 * The devich to configure through.
7411 * @param[in, out] matcher
7413 * @param[in, out] key
7414 * Flow matcher value.
7416 * Flow pattern to translate.
7419 * 0 on success, a negative errno value otherwise.
7422 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7423 void *key, const struct rte_flow_item *item)
7425 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7426 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7427 struct mlx5_priv *priv;
7430 mask = pid_m ? pid_m->id : 0xffff;
7431 id = pid_v ? pid_v->id : dev->data->port_id;
7432 priv = mlx5_port_to_eswitch_info(id, item == NULL);
7435 /* Translate to vport field or to metadata, depending on mode. */
7436 if (priv->vport_meta_mask)
7437 flow_dv_translate_item_meta_vport(matcher, key,
7438 priv->vport_meta_tag,
7439 priv->vport_meta_mask);
7441 flow_dv_translate_item_source_vport(matcher, key,
7442 priv->vport_id, mask);
7447 * Add ICMP6 item to matcher and to the value.
7449 * @param[in, out] matcher
7451 * @param[in, out] key
7452 * Flow matcher value.
7454 * Flow pattern to translate.
7456 * Item is inner pattern.
7459 flow_dv_translate_item_icmp6(void *matcher, void *key,
7460 const struct rte_flow_item *item,
7463 const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7464 const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7467 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7469 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7471 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7473 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7475 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7477 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7479 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7480 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7484 icmp6_m = &rte_flow_item_icmp6_mask;
7486 * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7487 * If only the protocol is specified, no need to match the frag.
7489 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7490 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7491 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7492 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7493 icmp6_v->type & icmp6_m->type);
7494 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7495 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7496 icmp6_v->code & icmp6_m->code);
7500 * Add ICMP item to matcher and to the value.
7502 * @param[in, out] matcher
7504 * @param[in, out] key
7505 * Flow matcher value.
7507 * Flow pattern to translate.
7509 * Item is inner pattern.
7512 flow_dv_translate_item_icmp(void *matcher, void *key,
7513 const struct rte_flow_item *item,
7516 const struct rte_flow_item_icmp *icmp_m = item->mask;
7517 const struct rte_flow_item_icmp *icmp_v = item->spec;
7518 uint32_t icmp_header_data_m = 0;
7519 uint32_t icmp_header_data_v = 0;
7522 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7524 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7526 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7528 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7530 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7532 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7534 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7535 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7539 icmp_m = &rte_flow_item_icmp_mask;
7541 * Force flow only to match the non-fragmented IPv4 ICMP packets.
7542 * If only the protocol is specified, no need to match the frag.
7544 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7545 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7546 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7547 icmp_m->hdr.icmp_type);
7548 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7549 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7550 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7551 icmp_m->hdr.icmp_code);
7552 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7553 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7554 icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7555 icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7556 if (icmp_header_data_m) {
7557 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7558 icmp_header_data_v |=
7559 rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7560 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7561 icmp_header_data_m);
7562 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7563 icmp_header_data_v & icmp_header_data_m);
7568 * Add GTP item to matcher and to the value.
7570 * @param[in, out] matcher
7572 * @param[in, out] key
7573 * Flow matcher value.
7575 * Flow pattern to translate.
7577 * Item is inner pattern.
7580 flow_dv_translate_item_gtp(void *matcher, void *key,
7581 const struct rte_flow_item *item, int inner)
7583 const struct rte_flow_item_gtp *gtp_m = item->mask;
7584 const struct rte_flow_item_gtp *gtp_v = item->spec;
7587 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7589 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7590 uint16_t dport = RTE_GTPU_UDP_PORT;
7593 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7595 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7597 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7599 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7601 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7602 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7603 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7608 gtp_m = &rte_flow_item_gtp_mask;
7609 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7610 gtp_m->v_pt_rsv_flags);
7611 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7612 gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7613 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7614 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7615 gtp_v->msg_type & gtp_m->msg_type);
7616 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7617 rte_be_to_cpu_32(gtp_m->teid));
7618 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7619 rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7623 * Add eCPRI item to matcher and to the value.
7626 * The devich to configure through.
7627 * @param[in, out] matcher
7629 * @param[in, out] key
7630 * Flow matcher value.
7632 * Flow pattern to translate.
7633 * @param[in] samples
7634 * Sample IDs to be used in the matching.
7637 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7638 void *key, const struct rte_flow_item *item)
7640 struct mlx5_priv *priv = dev->data->dev_private;
7641 const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7642 const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7643 void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7645 void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7653 ecpri_m = &rte_flow_item_ecpri_mask;
7655 * Maximal four DW samples are supported in a single matching now.
7656 * Two are used now for a eCPRI matching:
7657 * 1. Type: one byte, mask should be 0x00ff0000 in network order
7658 * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7661 if (!ecpri_m->hdr.common.u32)
7663 samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7664 /* Need to take the whole DW as the mask to fill the entry. */
7665 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7666 prog_sample_field_value_0);
7667 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7668 prog_sample_field_value_0);
7669 /* Already big endian (network order) in the header. */
7670 *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7671 *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7672 /* Sample#0, used for matching type, offset 0. */
7673 MLX5_SET(fte_match_set_misc4, misc4_m,
7674 prog_sample_field_id_0, samples[0]);
7675 /* It makes no sense to set the sample ID in the mask field. */
7676 MLX5_SET(fte_match_set_misc4, misc4_v,
7677 prog_sample_field_id_0, samples[0]);
7679 * Checking if message body part needs to be matched.
7680 * Some wildcard rules only matching type field should be supported.
7682 if (ecpri_m->hdr.dummy[0]) {
7683 switch (ecpri_v->hdr.common.type) {
7684 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7685 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7686 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7687 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7688 prog_sample_field_value_1);
7689 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7690 prog_sample_field_value_1);
7691 *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7692 *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7693 /* Sample#1, to match message body, offset 4. */
7694 MLX5_SET(fte_match_set_misc4, misc4_m,
7695 prog_sample_field_id_1, samples[1]);
7696 MLX5_SET(fte_match_set_misc4, misc4_v,
7697 prog_sample_field_id_1, samples[1]);
7700 /* Others, do not match any sample ID. */
7706 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7708 #define HEADER_IS_ZERO(match_criteria, headers) \
7709 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
7710 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7713 * Calculate flow matcher enable bitmap.
7715 * @param match_criteria
7716 * Pointer to flow matcher criteria.
7719 * Bitmap of enabled fields.
7722 flow_dv_matcher_enable(uint32_t *match_criteria)
7724 uint8_t match_criteria_enable;
7726 match_criteria_enable =
7727 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7728 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7729 match_criteria_enable |=
7730 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7731 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7732 match_criteria_enable |=
7733 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7734 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7735 match_criteria_enable |=
7736 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7737 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7738 match_criteria_enable |=
7739 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7740 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7741 match_criteria_enable |=
7742 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7743 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7744 return match_criteria_enable;
7751 * @param[in, out] dev
7752 * Pointer to rte_eth_dev structure.
7753 * @param[in] table_id
7756 * Direction of the table.
7757 * @param[in] transfer
7758 * E-Switch or NIC flow.
7760 * pointer to error structure.
7763 * Returns tables resource based on the index, NULL in case of failed.
7765 static struct mlx5_flow_tbl_resource *
7766 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7767 uint32_t table_id, uint8_t egress,
7769 struct rte_flow_error *error)
7771 struct mlx5_priv *priv = dev->data->dev_private;
7772 struct mlx5_dev_ctx_shared *sh = priv->sh;
7773 struct mlx5_flow_tbl_resource *tbl;
7774 union mlx5_flow_tbl_key table_key = {
7776 .table_id = table_id,
7778 .domain = !!transfer,
7779 .direction = !!egress,
7782 struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7784 struct mlx5_flow_tbl_data_entry *tbl_data;
7790 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7792 tbl = &tbl_data->tbl;
7793 rte_atomic32_inc(&tbl->refcnt);
7796 tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7798 rte_flow_error_set(error, ENOMEM,
7799 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7801 "cannot allocate flow table data entry");
7804 tbl_data->idx = idx;
7805 tbl = &tbl_data->tbl;
7806 pos = &tbl_data->entry;
7808 domain = sh->fdb_domain;
7810 domain = sh->tx_domain;
7812 domain = sh->rx_domain;
7813 ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7815 rte_flow_error_set(error, ENOMEM,
7816 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7817 NULL, "cannot create flow table object");
7818 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7822 * No multi-threads now, but still better to initialize the reference
7823 * count before insert it into the hash list.
7825 rte_atomic32_init(&tbl->refcnt);
7826 /* Jump action reference count is initialized here. */
7827 rte_atomic32_init(&tbl_data->jump.refcnt);
7828 pos->key = table_key.v64;
7829 ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7831 rte_flow_error_set(error, -ret,
7832 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7833 "cannot insert flow table data entry");
7834 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7835 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7837 rte_atomic32_inc(&tbl->refcnt);
7842 * Release a flow table.
7845 * Pointer to rte_eth_dev structure.
7847 * Table resource to be released.
7850 * Returns 0 if table was released, else return 1;
7853 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7854 struct mlx5_flow_tbl_resource *tbl)
7856 struct mlx5_priv *priv = dev->data->dev_private;
7857 struct mlx5_dev_ctx_shared *sh = priv->sh;
7858 struct mlx5_flow_tbl_data_entry *tbl_data =
7859 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7863 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7864 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7866 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7868 /* remove the entry from the hash list and free memory. */
7869 mlx5_hlist_remove(sh->flow_tbls, pos);
7870 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7878 * Register the flow matcher.
7880 * @param[in, out] dev
7881 * Pointer to rte_eth_dev structure.
7882 * @param[in, out] matcher
7883 * Pointer to flow matcher.
7884 * @param[in, out] key
7885 * Pointer to flow table key.
7886 * @parm[in, out] dev_flow
7887 * Pointer to the dev_flow.
7889 * pointer to error structure.
7892 * 0 on success otherwise -errno and errno is set.
7895 flow_dv_matcher_register(struct rte_eth_dev *dev,
7896 struct mlx5_flow_dv_matcher *matcher,
7897 union mlx5_flow_tbl_key *key,
7898 struct mlx5_flow *dev_flow,
7899 struct rte_flow_error *error)
7901 struct mlx5_priv *priv = dev->data->dev_private;
7902 struct mlx5_dev_ctx_shared *sh = priv->sh;
7903 struct mlx5_flow_dv_matcher *cache_matcher;
7904 struct mlx5dv_flow_matcher_attr dv_attr = {
7905 .type = IBV_FLOW_ATTR_NORMAL,
7906 .match_mask = (void *)&matcher->mask,
7908 struct mlx5_flow_tbl_resource *tbl;
7909 struct mlx5_flow_tbl_data_entry *tbl_data;
7912 tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7913 key->domain, error);
7915 return -rte_errno; /* No need to refill the error info */
7916 tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7917 /* Lookup from cache. */
7918 LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7919 if (matcher->crc == cache_matcher->crc &&
7920 matcher->priority == cache_matcher->priority &&
7921 !memcmp((const void *)matcher->mask.buf,
7922 (const void *)cache_matcher->mask.buf,
7923 cache_matcher->mask.size)) {
7925 "%s group %u priority %hd use %s "
7926 "matcher %p: refcnt %d++",
7927 key->domain ? "FDB" : "NIC", key->table_id,
7928 cache_matcher->priority,
7929 key->direction ? "tx" : "rx",
7930 (void *)cache_matcher,
7931 rte_atomic32_read(&cache_matcher->refcnt));
7932 rte_atomic32_inc(&cache_matcher->refcnt);
7933 dev_flow->handle->dvh.matcher = cache_matcher;
7934 /* old matcher should not make the table ref++. */
7935 flow_dv_tbl_resource_release(dev, tbl);
7939 /* Register new matcher. */
7940 cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
7942 if (!cache_matcher) {
7943 flow_dv_tbl_resource_release(dev, tbl);
7944 return rte_flow_error_set(error, ENOMEM,
7945 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7946 "cannot allocate matcher memory");
7948 *cache_matcher = *matcher;
7949 dv_attr.match_criteria_enable =
7950 flow_dv_matcher_enable(cache_matcher->mask.buf);
7951 dv_attr.priority = matcher->priority;
7953 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7954 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
7955 &cache_matcher->matcher_object);
7957 mlx5_free(cache_matcher);
7958 #ifdef HAVE_MLX5DV_DR
7959 flow_dv_tbl_resource_release(dev, tbl);
7961 return rte_flow_error_set(error, ENOMEM,
7962 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7963 NULL, "cannot create matcher");
7965 /* Save the table information */
7966 cache_matcher->tbl = tbl;
7967 rte_atomic32_init(&cache_matcher->refcnt);
7968 /* only matcher ref++, table ref++ already done above in get API. */
7969 rte_atomic32_inc(&cache_matcher->refcnt);
7970 LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7971 dev_flow->handle->dvh.matcher = cache_matcher;
7972 DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7973 key->domain ? "FDB" : "NIC", key->table_id,
7974 cache_matcher->priority,
7975 key->direction ? "tx" : "rx", (void *)cache_matcher,
7976 rte_atomic32_read(&cache_matcher->refcnt));
7981 * Find existing tag resource or create and register a new one.
7983 * @param dev[in, out]
7984 * Pointer to rte_eth_dev structure.
7985 * @param[in, out] tag_be24
7986 * Tag value in big endian then R-shift 8.
7987 * @parm[in, out] dev_flow
7988 * Pointer to the dev_flow.
7990 * pointer to error structure.
7993 * 0 on success otherwise -errno and errno is set.
7996 flow_dv_tag_resource_register
7997 (struct rte_eth_dev *dev,
7999 struct mlx5_flow *dev_flow,
8000 struct rte_flow_error *error)
8002 struct mlx5_priv *priv = dev->data->dev_private;
8003 struct mlx5_dev_ctx_shared *sh = priv->sh;
8004 struct mlx5_flow_dv_tag_resource *cache_resource;
8005 struct mlx5_hlist_entry *entry;
8008 /* Lookup a matching resource from cache. */
8009 entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8011 cache_resource = container_of
8012 (entry, struct mlx5_flow_dv_tag_resource, entry);
8013 rte_atomic32_inc(&cache_resource->refcnt);
8014 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8015 dev_flow->dv.tag_resource = cache_resource;
8016 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8017 (void *)cache_resource,
8018 rte_atomic32_read(&cache_resource->refcnt));
8021 /* Register new resource. */
8022 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8023 &dev_flow->handle->dvh.rix_tag);
8024 if (!cache_resource)
8025 return rte_flow_error_set(error, ENOMEM,
8026 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8027 "cannot allocate resource memory");
8028 cache_resource->entry.key = (uint64_t)tag_be24;
8029 ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8030 &cache_resource->action);
8032 mlx5_free(cache_resource);
8033 return rte_flow_error_set(error, ENOMEM,
8034 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8035 NULL, "cannot create action");
8037 rte_atomic32_init(&cache_resource->refcnt);
8038 rte_atomic32_inc(&cache_resource->refcnt);
8039 if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8040 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8041 mlx5_free(cache_resource);
8042 return rte_flow_error_set(error, EEXIST,
8043 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8044 NULL, "cannot insert tag");
8046 dev_flow->dv.tag_resource = cache_resource;
8047 DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8048 (void *)cache_resource,
8049 rte_atomic32_read(&cache_resource->refcnt));
8057 * Pointer to Ethernet device.
8062 * 1 while a reference on it exists, 0 when freed.
8065 flow_dv_tag_release(struct rte_eth_dev *dev,
8068 struct mlx5_priv *priv = dev->data->dev_private;
8069 struct mlx5_dev_ctx_shared *sh = priv->sh;
8070 struct mlx5_flow_dv_tag_resource *tag;
8072 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8075 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8076 dev->data->port_id, (void *)tag,
8077 rte_atomic32_read(&tag->refcnt));
8078 if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8079 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8080 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8081 DRV_LOG(DEBUG, "port %u tag %p: removed",
8082 dev->data->port_id, (void *)tag);
8083 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8090 * Translate port ID action to vport.
8093 * Pointer to rte_eth_dev structure.
8095 * Pointer to the port ID action.
8096 * @param[out] dst_port_id
8097 * The target port ID.
8099 * Pointer to the error structure.
8102 * 0 on success, a negative errno value otherwise and rte_errno is set.
8105 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8106 const struct rte_flow_action *action,
8107 uint32_t *dst_port_id,
8108 struct rte_flow_error *error)
8111 struct mlx5_priv *priv;
8112 const struct rte_flow_action_port_id *conf =
8113 (const struct rte_flow_action_port_id *)action->conf;
8115 port = conf->original ? dev->data->port_id : conf->id;
8116 priv = mlx5_port_to_eswitch_info(port, false);
8118 return rte_flow_error_set(error, -rte_errno,
8119 RTE_FLOW_ERROR_TYPE_ACTION,
8121 "No eswitch info was found for port");
8122 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8124 * This parameter is transferred to
8125 * mlx5dv_dr_action_create_dest_ib_port().
8127 *dst_port_id = priv->dev_port;
8130 * Legacy mode, no LAG configurations is supported.
8131 * This parameter is transferred to
8132 * mlx5dv_dr_action_create_dest_vport().
8134 *dst_port_id = priv->vport_id;
8140 * Create a counter with aging configuration.
8143 * Pointer to rte_eth_dev structure.
8145 * Pointer to the counter action configuration.
8147 * Pointer to the aging action configuration.
8150 * Index to flow counter on success, 0 otherwise.
8153 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8154 struct mlx5_flow *dev_flow,
8155 const struct rte_flow_action_count *count,
8156 const struct rte_flow_action_age *age)
8159 struct mlx5_age_param *age_param;
8161 counter = flow_dv_counter_alloc(dev,
8162 count ? count->shared : 0,
8163 count ? count->id : 0,
8164 dev_flow->dv.group, !!age);
8165 if (!counter || age == NULL)
8167 age_param = flow_dv_counter_idx_get_age(dev, counter);
8169 * The counter age accuracy may have a bit delay. Have 3/4
8170 * second bias on the timeount in order to let it age in time.
8172 age_param->context = age->context ? age->context :
8173 (void *)(uintptr_t)(dev_flow->flow_idx);
8175 * The counter age accuracy may have a bit delay. Have 3/4
8176 * second bias on the timeount in order to let it age in time.
8178 age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
8179 /* Set expire time in unit of 0.1 sec. */
8180 age_param->port_id = dev->data->port_id;
8181 age_param->expire = age_param->timeout +
8182 rte_rdtsc() / (rte_get_tsc_hz() / 10);
8183 rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
8187 * Add Tx queue matcher
8190 * Pointer to the dev struct.
8191 * @param[in, out] matcher
8193 * @param[in, out] key
8194 * Flow matcher value.
8196 * Flow pattern to translate.
8198 * Item is inner pattern.
8201 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8202 void *matcher, void *key,
8203 const struct rte_flow_item *item)
8205 const struct mlx5_rte_flow_item_tx_queue *queue_m;
8206 const struct mlx5_rte_flow_item_tx_queue *queue_v;
8208 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8210 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8211 struct mlx5_txq_ctrl *txq;
8215 queue_m = (const void *)item->mask;
8218 queue_v = (const void *)item->spec;
8221 txq = mlx5_txq_get(dev, queue_v->queue);
8224 queue = txq->obj->sq->id;
8225 MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8226 MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8227 queue & queue_m->queue);
8228 mlx5_txq_release(dev, queue_v->queue);
8232 * Set the hash fields according to the @p flow information.
8234 * @param[in] dev_flow
8235 * Pointer to the mlx5_flow.
8236 * @param[in] rss_desc
8237 * Pointer to the mlx5_flow_rss_desc.
8240 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8241 struct mlx5_flow_rss_desc *rss_desc)
8243 uint64_t items = dev_flow->handle->layers;
8245 uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8247 dev_flow->hash_fields = 0;
8248 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8249 if (rss_desc->level >= 2) {
8250 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8254 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8255 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8256 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8257 if (rss_types & ETH_RSS_L3_SRC_ONLY)
8258 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8259 else if (rss_types & ETH_RSS_L3_DST_ONLY)
8260 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8262 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8264 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8265 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8266 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8267 if (rss_types & ETH_RSS_L3_SRC_ONLY)
8268 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8269 else if (rss_types & ETH_RSS_L3_DST_ONLY)
8270 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8272 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8275 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8276 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8277 if (rss_types & ETH_RSS_UDP) {
8278 if (rss_types & ETH_RSS_L4_SRC_ONLY)
8279 dev_flow->hash_fields |=
8280 IBV_RX_HASH_SRC_PORT_UDP;
8281 else if (rss_types & ETH_RSS_L4_DST_ONLY)
8282 dev_flow->hash_fields |=
8283 IBV_RX_HASH_DST_PORT_UDP;
8285 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8287 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8288 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8289 if (rss_types & ETH_RSS_TCP) {
8290 if (rss_types & ETH_RSS_L4_SRC_ONLY)
8291 dev_flow->hash_fields |=
8292 IBV_RX_HASH_SRC_PORT_TCP;
8293 else if (rss_types & ETH_RSS_L4_DST_ONLY)
8294 dev_flow->hash_fields |=
8295 IBV_RX_HASH_DST_PORT_TCP;
8297 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8303 * Fill the flow with DV spec, lock free
8304 * (mutex should be acquired by caller).
8307 * Pointer to rte_eth_dev structure.
8308 * @param[in, out] dev_flow
8309 * Pointer to the sub flow.
8311 * Pointer to the flow attributes.
8313 * Pointer to the list of items.
8314 * @param[in] actions
8315 * Pointer to the list of actions.
8317 * Pointer to the error structure.
8320 * 0 on success, a negative errno value otherwise and rte_errno is set.
8323 __flow_dv_translate(struct rte_eth_dev *dev,
8324 struct mlx5_flow *dev_flow,
8325 const struct rte_flow_attr *attr,
8326 const struct rte_flow_item items[],
8327 const struct rte_flow_action actions[],
8328 struct rte_flow_error *error)
8330 struct mlx5_priv *priv = dev->data->dev_private;
8331 struct mlx5_dev_config *dev_conf = &priv->config;
8332 struct rte_flow *flow = dev_flow->flow;
8333 struct mlx5_flow_handle *handle = dev_flow->handle;
8334 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8336 [!!priv->flow_nested_idx];
8337 uint64_t item_flags = 0;
8338 uint64_t last_item = 0;
8339 uint64_t action_flags = 0;
8340 uint64_t priority = attr->priority;
8341 struct mlx5_flow_dv_matcher matcher = {
8343 .size = sizeof(matcher.mask.buf) -
8344 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
8348 bool actions_end = false;
8350 struct mlx5_flow_dv_modify_hdr_resource res;
8351 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
8352 sizeof(struct mlx5_modification_cmd) *
8353 (MLX5_MAX_MODIFY_NUM + 1)];
8355 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
8356 const struct rte_flow_action_count *count = NULL;
8357 const struct rte_flow_action_age *age = NULL;
8358 union flow_dv_attr flow_attr = { .attr = 0 };
8360 union mlx5_flow_tbl_key tbl_key;
8361 uint32_t modify_action_position = UINT32_MAX;
8362 void *match_mask = matcher.mask.buf;
8363 void *match_value = dev_flow->dv.value.buf;
8364 uint8_t next_protocol = 0xff;
8365 struct rte_vlan_hdr vlan = { 0 };
8369 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
8370 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
8371 ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
8372 !!priv->fdb_def_rule, &table, error);
8375 dev_flow->dv.group = table;
8377 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
8378 if (priority == MLX5_FLOW_PRIO_RSVD)
8379 priority = dev_conf->flow_prio - 1;
8380 /* number of actions must be set to 0 in case of dirty stack. */
8381 mhdr_res->actions_num = 0;
8382 for (; !actions_end ; actions++) {
8383 const struct rte_flow_action_queue *queue;
8384 const struct rte_flow_action_rss *rss;
8385 const struct rte_flow_action *action = actions;
8386 const uint8_t *rss_key;
8387 const struct rte_flow_action_jump *jump_data;
8388 const struct rte_flow_action_meter *mtr;
8389 struct mlx5_flow_tbl_resource *tbl;
8390 uint32_t port_id = 0;
8391 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
8392 int action_type = actions->type;
8393 const struct rte_flow_action *found_action = NULL;
8394 struct mlx5_flow_meter *fm = NULL;
8396 if (!mlx5_flow_os_action_supported(action_type))
8397 return rte_flow_error_set(error, ENOTSUP,
8398 RTE_FLOW_ERROR_TYPE_ACTION,
8400 "action not supported");
8401 switch (action_type) {
8402 case RTE_FLOW_ACTION_TYPE_VOID:
8404 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8405 if (flow_dv_translate_action_port_id(dev, action,
8408 port_id_resource.port_id = port_id;
8409 MLX5_ASSERT(!handle->rix_port_id_action);
8410 if (flow_dv_port_id_action_resource_register
8411 (dev, &port_id_resource, dev_flow, error))
8413 dev_flow->dv.actions[actions_n++] =
8414 dev_flow->dv.port_id_action->action;
8415 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8416 dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
8418 case RTE_FLOW_ACTION_TYPE_FLAG:
8419 action_flags |= MLX5_FLOW_ACTION_FLAG;
8420 dev_flow->handle->mark = 1;
8421 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8422 struct rte_flow_action_mark mark = {
8423 .id = MLX5_FLOW_MARK_DEFAULT,
8426 if (flow_dv_convert_action_mark(dev, &mark,
8430 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8433 tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
8435 * Only one FLAG or MARK is supported per device flow
8436 * right now. So the pointer to the tag resource must be
8437 * zero before the register process.
8439 MLX5_ASSERT(!handle->dvh.rix_tag);
8440 if (flow_dv_tag_resource_register(dev, tag_be,
8443 MLX5_ASSERT(dev_flow->dv.tag_resource);
8444 dev_flow->dv.actions[actions_n++] =
8445 dev_flow->dv.tag_resource->action;
8447 case RTE_FLOW_ACTION_TYPE_MARK:
8448 action_flags |= MLX5_FLOW_ACTION_MARK;
8449 dev_flow->handle->mark = 1;
8450 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8451 const struct rte_flow_action_mark *mark =
8452 (const struct rte_flow_action_mark *)
8455 if (flow_dv_convert_action_mark(dev, mark,
8459 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8463 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
8464 /* Legacy (non-extensive) MARK action. */
8465 tag_be = mlx5_flow_mark_set
8466 (((const struct rte_flow_action_mark *)
8467 (actions->conf))->id);
8468 MLX5_ASSERT(!handle->dvh.rix_tag);
8469 if (flow_dv_tag_resource_register(dev, tag_be,
8472 MLX5_ASSERT(dev_flow->dv.tag_resource);
8473 dev_flow->dv.actions[actions_n++] =
8474 dev_flow->dv.tag_resource->action;
8476 case RTE_FLOW_ACTION_TYPE_SET_META:
8477 if (flow_dv_convert_action_set_meta
8478 (dev, mhdr_res, attr,
8479 (const struct rte_flow_action_set_meta *)
8480 actions->conf, error))
8482 action_flags |= MLX5_FLOW_ACTION_SET_META;
8484 case RTE_FLOW_ACTION_TYPE_SET_TAG:
8485 if (flow_dv_convert_action_set_tag
8487 (const struct rte_flow_action_set_tag *)
8488 actions->conf, error))
8490 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8492 case RTE_FLOW_ACTION_TYPE_DROP:
8493 action_flags |= MLX5_FLOW_ACTION_DROP;
8494 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8496 case RTE_FLOW_ACTION_TYPE_QUEUE:
8497 queue = actions->conf;
8498 rss_desc->queue_num = 1;
8499 rss_desc->queue[0] = queue->index;
8500 action_flags |= MLX5_FLOW_ACTION_QUEUE;
8501 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8503 case RTE_FLOW_ACTION_TYPE_RSS:
8504 rss = actions->conf;
8505 memcpy(rss_desc->queue, rss->queue,
8506 rss->queue_num * sizeof(uint16_t));
8507 rss_desc->queue_num = rss->queue_num;
8508 /* NULL RSS key indicates default RSS key. */
8509 rss_key = !rss->key ? rss_hash_default_key : rss->key;
8510 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8512 * rss->level and rss.types should be set in advance
8513 * when expanding items for RSS.
8515 action_flags |= MLX5_FLOW_ACTION_RSS;
8516 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8518 case RTE_FLOW_ACTION_TYPE_AGE:
8519 case RTE_FLOW_ACTION_TYPE_COUNT:
8520 if (!dev_conf->devx) {
8521 return rte_flow_error_set
8523 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8525 "count action not supported");
8527 /* Save information first, will apply later. */
8528 if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8529 count = action->conf;
8532 action_flags |= MLX5_FLOW_ACTION_COUNT;
8534 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8535 dev_flow->dv.actions[actions_n++] =
8536 priv->sh->pop_vlan_action;
8537 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8539 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8540 if (!(action_flags &
8541 MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8542 flow_dev_get_vlan_info_from_items(items, &vlan);
8543 vlan.eth_proto = rte_be_to_cpu_16
8544 ((((const struct rte_flow_action_of_push_vlan *)
8545 actions->conf)->ethertype));
8546 found_action = mlx5_flow_find_action
8548 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8550 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8551 found_action = mlx5_flow_find_action
8553 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8555 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8556 if (flow_dv_create_action_push_vlan
8557 (dev, attr, &vlan, dev_flow, error))
8559 dev_flow->dv.actions[actions_n++] =
8560 dev_flow->dv.push_vlan_res->action;
8561 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8563 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8564 /* of_vlan_push action handled this action */
8565 MLX5_ASSERT(action_flags &
8566 MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8568 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8569 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8571 flow_dev_get_vlan_info_from_items(items, &vlan);
8572 mlx5_update_vlan_vid_pcp(actions, &vlan);
8573 /* If no VLAN push - this is a modify header action */
8574 if (flow_dv_convert_action_modify_vlan_vid
8575 (mhdr_res, actions, error))
8577 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8579 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8580 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8581 if (flow_dv_create_action_l2_encap(dev, actions,
8586 dev_flow->dv.actions[actions_n++] =
8587 dev_flow->dv.encap_decap->action;
8588 action_flags |= MLX5_FLOW_ACTION_ENCAP;
8590 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8591 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8592 if (flow_dv_create_action_l2_decap(dev, dev_flow,
8596 dev_flow->dv.actions[actions_n++] =
8597 dev_flow->dv.encap_decap->action;
8598 action_flags |= MLX5_FLOW_ACTION_DECAP;
8600 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8601 /* Handle encap with preceding decap. */
8602 if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8603 if (flow_dv_create_action_raw_encap
8604 (dev, actions, dev_flow, attr, error))
8606 dev_flow->dv.actions[actions_n++] =
8607 dev_flow->dv.encap_decap->action;
8609 /* Handle encap without preceding decap. */
8610 if (flow_dv_create_action_l2_encap
8611 (dev, actions, dev_flow, attr->transfer,
8614 dev_flow->dv.actions[actions_n++] =
8615 dev_flow->dv.encap_decap->action;
8617 action_flags |= MLX5_FLOW_ACTION_ENCAP;
8619 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8620 while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8622 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8623 if (flow_dv_create_action_l2_decap
8624 (dev, dev_flow, attr->transfer, error))
8626 dev_flow->dv.actions[actions_n++] =
8627 dev_flow->dv.encap_decap->action;
8629 /* If decap is followed by encap, handle it at encap. */
8630 action_flags |= MLX5_FLOW_ACTION_DECAP;
8632 case RTE_FLOW_ACTION_TYPE_JUMP:
8633 jump_data = action->conf;
8634 ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8636 !!priv->fdb_def_rule,
8640 tbl = flow_dv_tbl_resource_get(dev, table,
8642 attr->transfer, error);
8644 return rte_flow_error_set
8646 RTE_FLOW_ERROR_TYPE_ACTION,
8648 "cannot create jump action.");
8649 if (flow_dv_jump_tbl_resource_register
8650 (dev, tbl, dev_flow, error)) {
8651 flow_dv_tbl_resource_release(dev, tbl);
8652 return rte_flow_error_set
8654 RTE_FLOW_ERROR_TYPE_ACTION,
8656 "cannot create jump action.");
8658 dev_flow->dv.actions[actions_n++] =
8659 dev_flow->dv.jump->action;
8660 action_flags |= MLX5_FLOW_ACTION_JUMP;
8661 dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8663 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8664 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8665 if (flow_dv_convert_action_modify_mac
8666 (mhdr_res, actions, error))
8668 action_flags |= actions->type ==
8669 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8670 MLX5_FLOW_ACTION_SET_MAC_SRC :
8671 MLX5_FLOW_ACTION_SET_MAC_DST;
8673 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8674 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8675 if (flow_dv_convert_action_modify_ipv4
8676 (mhdr_res, actions, error))
8678 action_flags |= actions->type ==
8679 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8680 MLX5_FLOW_ACTION_SET_IPV4_SRC :
8681 MLX5_FLOW_ACTION_SET_IPV4_DST;
8683 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8684 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8685 if (flow_dv_convert_action_modify_ipv6
8686 (mhdr_res, actions, error))
8688 action_flags |= actions->type ==
8689 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8690 MLX5_FLOW_ACTION_SET_IPV6_SRC :
8691 MLX5_FLOW_ACTION_SET_IPV6_DST;
8693 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8694 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8695 if (flow_dv_convert_action_modify_tp
8696 (mhdr_res, actions, items,
8697 &flow_attr, dev_flow, !!(action_flags &
8698 MLX5_FLOW_ACTION_DECAP), error))
8700 action_flags |= actions->type ==
8701 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8702 MLX5_FLOW_ACTION_SET_TP_SRC :
8703 MLX5_FLOW_ACTION_SET_TP_DST;
8705 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8706 if (flow_dv_convert_action_modify_dec_ttl
8707 (mhdr_res, items, &flow_attr, dev_flow,
8709 MLX5_FLOW_ACTION_DECAP), error))
8711 action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8713 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8714 if (flow_dv_convert_action_modify_ttl
8715 (mhdr_res, actions, items, &flow_attr,
8716 dev_flow, !!(action_flags &
8717 MLX5_FLOW_ACTION_DECAP), error))
8719 action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8721 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8722 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8723 if (flow_dv_convert_action_modify_tcp_seq
8724 (mhdr_res, actions, error))
8726 action_flags |= actions->type ==
8727 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8728 MLX5_FLOW_ACTION_INC_TCP_SEQ :
8729 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8732 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8733 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8734 if (flow_dv_convert_action_modify_tcp_ack
8735 (mhdr_res, actions, error))
8737 action_flags |= actions->type ==
8738 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8739 MLX5_FLOW_ACTION_INC_TCP_ACK :
8740 MLX5_FLOW_ACTION_DEC_TCP_ACK;
8742 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8743 if (flow_dv_convert_action_set_reg
8744 (mhdr_res, actions, error))
8746 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8748 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8749 if (flow_dv_convert_action_copy_mreg
8750 (dev, mhdr_res, actions, error))
8752 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8754 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8755 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8756 dev_flow->handle->fate_action =
8757 MLX5_FLOW_FATE_DEFAULT_MISS;
8759 case RTE_FLOW_ACTION_TYPE_METER:
8760 mtr = actions->conf;
8762 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8765 return rte_flow_error_set(error,
8767 RTE_FLOW_ERROR_TYPE_ACTION,
8770 "or invalid parameters");
8771 flow->meter = fm->idx;
8773 /* Set the meter action. */
8775 fm = mlx5_ipool_get(priv->sh->ipool
8776 [MLX5_IPOOL_MTR], flow->meter);
8778 return rte_flow_error_set(error,
8780 RTE_FLOW_ERROR_TYPE_ACTION,
8783 "or invalid parameters");
8785 dev_flow->dv.actions[actions_n++] =
8786 fm->mfts->meter_action;
8787 action_flags |= MLX5_FLOW_ACTION_METER;
8789 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8790 if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8793 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8795 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8796 if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8799 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8801 case RTE_FLOW_ACTION_TYPE_END:
8803 if (mhdr_res->actions_num) {
8804 /* create modify action if needed. */
8805 if (flow_dv_modify_hdr_resource_register
8806 (dev, mhdr_res, dev_flow, error))
8808 dev_flow->dv.actions[modify_action_position] =
8809 handle->dvh.modify_hdr->action;
8811 if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8813 flow_dv_translate_create_counter(dev,
8814 dev_flow, count, age);
8817 return rte_flow_error_set
8819 RTE_FLOW_ERROR_TYPE_ACTION,
8821 "cannot create counter"
8823 dev_flow->dv.actions[actions_n++] =
8824 (flow_dv_counter_get_by_idx(dev,
8825 flow->counter, NULL))->action;
8831 if (mhdr_res->actions_num &&
8832 modify_action_position == UINT32_MAX)
8833 modify_action_position = actions_n++;
8835 dev_flow->dv.actions_n = actions_n;
8836 dev_flow->act_flags = action_flags;
8837 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8838 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8839 int item_type = items->type;
8841 if (!mlx5_flow_os_item_supported(item_type))
8842 return rte_flow_error_set(error, ENOTSUP,
8843 RTE_FLOW_ERROR_TYPE_ITEM,
8844 NULL, "item not supported");
8845 switch (item_type) {
8846 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8847 flow_dv_translate_item_port_id(dev, match_mask,
8848 match_value, items);
8849 last_item = MLX5_FLOW_ITEM_PORT_ID;
8851 case RTE_FLOW_ITEM_TYPE_ETH:
8852 flow_dv_translate_item_eth(match_mask, match_value,
8854 dev_flow->dv.group);
8855 matcher.priority = action_flags &
8856 MLX5_FLOW_ACTION_DEFAULT_MISS &&
8857 !dev_flow->external ?
8858 MLX5_PRIORITY_MAP_L3 :
8859 MLX5_PRIORITY_MAP_L2;
8860 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8861 MLX5_FLOW_LAYER_OUTER_L2;
8863 case RTE_FLOW_ITEM_TYPE_VLAN:
8864 flow_dv_translate_item_vlan(dev_flow,
8865 match_mask, match_value,
8867 dev_flow->dv.group);
8868 matcher.priority = MLX5_PRIORITY_MAP_L2;
8869 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8870 MLX5_FLOW_LAYER_INNER_VLAN) :
8871 (MLX5_FLOW_LAYER_OUTER_L2 |
8872 MLX5_FLOW_LAYER_OUTER_VLAN);
8874 case RTE_FLOW_ITEM_TYPE_IPV4:
8875 mlx5_flow_tunnel_ip_check(items, next_protocol,
8876 &item_flags, &tunnel);
8877 flow_dv_translate_item_ipv4(match_mask, match_value,
8878 items, item_flags, tunnel,
8879 dev_flow->dv.group);
8880 matcher.priority = MLX5_PRIORITY_MAP_L3;
8881 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8882 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8883 if (items->mask != NULL &&
8884 ((const struct rte_flow_item_ipv4 *)
8885 items->mask)->hdr.next_proto_id) {
8887 ((const struct rte_flow_item_ipv4 *)
8888 (items->spec))->hdr.next_proto_id;
8890 ((const struct rte_flow_item_ipv4 *)
8891 (items->mask))->hdr.next_proto_id;
8893 /* Reset for inner layer. */
8894 next_protocol = 0xff;
8897 case RTE_FLOW_ITEM_TYPE_IPV6:
8898 mlx5_flow_tunnel_ip_check(items, next_protocol,
8899 &item_flags, &tunnel);
8900 flow_dv_translate_item_ipv6(match_mask, match_value,
8901 items, item_flags, tunnel,
8902 dev_flow->dv.group);
8903 matcher.priority = MLX5_PRIORITY_MAP_L3;
8904 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8905 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8906 if (items->mask != NULL &&
8907 ((const struct rte_flow_item_ipv6 *)
8908 items->mask)->hdr.proto) {
8910 ((const struct rte_flow_item_ipv6 *)
8911 items->spec)->hdr.proto;
8913 ((const struct rte_flow_item_ipv6 *)
8914 items->mask)->hdr.proto;
8916 /* Reset for inner layer. */
8917 next_protocol = 0xff;
8920 case RTE_FLOW_ITEM_TYPE_TCP:
8921 flow_dv_translate_item_tcp(match_mask, match_value,
8923 matcher.priority = MLX5_PRIORITY_MAP_L4;
8924 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8925 MLX5_FLOW_LAYER_OUTER_L4_TCP;
8927 case RTE_FLOW_ITEM_TYPE_UDP:
8928 flow_dv_translate_item_udp(match_mask, match_value,
8930 matcher.priority = MLX5_PRIORITY_MAP_L4;
8931 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8932 MLX5_FLOW_LAYER_OUTER_L4_UDP;
8934 case RTE_FLOW_ITEM_TYPE_GRE:
8935 flow_dv_translate_item_gre(match_mask, match_value,
8937 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
8938 last_item = MLX5_FLOW_LAYER_GRE;
8940 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8941 flow_dv_translate_item_gre_key(match_mask,
8942 match_value, items);
8943 last_item = MLX5_FLOW_LAYER_GRE_KEY;
8945 case RTE_FLOW_ITEM_TYPE_NVGRE:
8946 flow_dv_translate_item_nvgre(match_mask, match_value,
8948 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
8949 last_item = MLX5_FLOW_LAYER_GRE;
8951 case RTE_FLOW_ITEM_TYPE_VXLAN:
8952 flow_dv_translate_item_vxlan(match_mask, match_value,
8954 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
8955 last_item = MLX5_FLOW_LAYER_VXLAN;
8957 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8958 flow_dv_translate_item_vxlan_gpe(match_mask,
8961 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
8962 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8964 case RTE_FLOW_ITEM_TYPE_GENEVE:
8965 flow_dv_translate_item_geneve(match_mask, match_value,
8967 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
8968 last_item = MLX5_FLOW_LAYER_GENEVE;
8970 case RTE_FLOW_ITEM_TYPE_MPLS:
8971 flow_dv_translate_item_mpls(match_mask, match_value,
8972 items, last_item, tunnel);
8973 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
8974 last_item = MLX5_FLOW_LAYER_MPLS;
8976 case RTE_FLOW_ITEM_TYPE_MARK:
8977 flow_dv_translate_item_mark(dev, match_mask,
8978 match_value, items);
8979 last_item = MLX5_FLOW_ITEM_MARK;
8981 case RTE_FLOW_ITEM_TYPE_META:
8982 flow_dv_translate_item_meta(dev, match_mask,
8983 match_value, attr, items);
8984 last_item = MLX5_FLOW_ITEM_METADATA;
8986 case RTE_FLOW_ITEM_TYPE_ICMP:
8987 flow_dv_translate_item_icmp(match_mask, match_value,
8989 last_item = MLX5_FLOW_LAYER_ICMP;
8991 case RTE_FLOW_ITEM_TYPE_ICMP6:
8992 flow_dv_translate_item_icmp6(match_mask, match_value,
8994 last_item = MLX5_FLOW_LAYER_ICMP6;
8996 case RTE_FLOW_ITEM_TYPE_TAG:
8997 flow_dv_translate_item_tag(dev, match_mask,
8998 match_value, items);
8999 last_item = MLX5_FLOW_ITEM_TAG;
9001 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
9002 flow_dv_translate_mlx5_item_tag(dev, match_mask,
9003 match_value, items);
9004 last_item = MLX5_FLOW_ITEM_TAG;
9006 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
9007 flow_dv_translate_item_tx_queue(dev, match_mask,
9010 last_item = MLX5_FLOW_ITEM_TX_QUEUE;
9012 case RTE_FLOW_ITEM_TYPE_GTP:
9013 flow_dv_translate_item_gtp(match_mask, match_value,
9015 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9016 last_item = MLX5_FLOW_LAYER_GTP;
9018 case RTE_FLOW_ITEM_TYPE_ECPRI:
9019 if (!mlx5_flex_parser_ecpri_exist(dev)) {
9020 /* Create it only the first time to be used. */
9021 ret = mlx5_flex_parser_ecpri_alloc(dev);
9023 return rte_flow_error_set
9025 RTE_FLOW_ERROR_TYPE_ITEM,
9027 "cannot create eCPRI parser");
9029 /* Adjust the length matcher and device flow value. */
9030 matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
9031 dev_flow->dv.value.size =
9032 MLX5_ST_SZ_BYTES(fte_match_param);
9033 flow_dv_translate_item_ecpri(dev, match_mask,
9034 match_value, items);
9035 /* No other protocol should follow eCPRI layer. */
9036 last_item = MLX5_FLOW_LAYER_ECPRI;
9041 item_flags |= last_item;
9044 * When E-Switch mode is enabled, we have two cases where we need to
9045 * set the source port manually.
9046 * The first one, is in case of Nic steering rule, and the second is
9047 * E-Switch rule where no port_id item was found. In both cases
9048 * the source port is set according the current port in use.
9050 if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
9051 (priv->representor || priv->master)) {
9052 if (flow_dv_translate_item_port_id(dev, match_mask,
9056 #ifdef RTE_LIBRTE_MLX5_DEBUG
9057 MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
9058 dev_flow->dv.value.buf));
9061 * Layers may be already initialized from prefix flow if this dev_flow
9062 * is the suffix flow.
9064 handle->layers |= item_flags;
9065 if (action_flags & MLX5_FLOW_ACTION_RSS)
9066 flow_dv_hashfields_set(dev_flow, rss_desc);
9067 /* Register matcher. */
9068 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
9070 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
9072 /* reserved field no needs to be set to 0 here. */
9073 tbl_key.domain = attr->transfer;
9074 tbl_key.direction = attr->egress;
9075 tbl_key.table_id = dev_flow->dv.group;
9076 if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
9082 * Apply the flow to the NIC, lock free,
9083 * (mutex should be acquired by caller).
9086 * Pointer to the Ethernet device structure.
9087 * @param[in, out] flow
9088 * Pointer to flow structure.
9090 * Pointer to error structure.
9093 * 0 on success, a negative errno value otherwise and rte_errno is set.
9096 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
9097 struct rte_flow_error *error)
9099 struct mlx5_flow_dv_workspace *dv;
9100 struct mlx5_flow_handle *dh;
9101 struct mlx5_flow_handle_dv *dv_h;
9102 struct mlx5_flow *dev_flow;
9103 struct mlx5_priv *priv = dev->data->dev_private;
9104 uint32_t handle_idx;
9109 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
9110 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
9112 dh = dev_flow->handle;
9115 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
9117 dv->actions[n++] = priv->sh->esw_drop_action;
9119 struct mlx5_hrxq *drop_hrxq;
9120 drop_hrxq = mlx5_drop_action_create(dev);
9124 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9126 "cannot get drop hash queue");
9130 * Drop queues will be released by the specify
9131 * mlx5_drop_action_destroy() function. Assign
9132 * the special index to hrxq to mark the queue
9133 * has been allocated.
9135 dh->rix_hrxq = UINT32_MAX;
9136 dv->actions[n++] = drop_hrxq->action;
9138 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
9139 struct mlx5_hrxq *hrxq;
9141 struct mlx5_flow_rss_desc *rss_desc =
9142 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
9143 [!!priv->flow_nested_idx];
9145 MLX5_ASSERT(rss_desc->queue_num);
9146 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
9147 MLX5_RSS_HASH_KEY_LEN,
9148 dev_flow->hash_fields,
9150 rss_desc->queue_num);
9152 hrxq_idx = mlx5_hrxq_new
9153 (dev, rss_desc->key,
9154 MLX5_RSS_HASH_KEY_LEN,
9155 dev_flow->hash_fields,
9157 rss_desc->queue_num,
9159 MLX5_FLOW_LAYER_TUNNEL));
9161 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
9166 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9167 "cannot get hash queue");
9170 dh->rix_hrxq = hrxq_idx;
9171 dv->actions[n++] = hrxq->action;
9172 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
9173 if (flow_dv_default_miss_resource_register
9177 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9178 "cannot create default miss resource");
9179 goto error_default_miss;
9181 dh->rix_default_fate = MLX5_FLOW_FATE_DEFAULT_MISS;
9182 dv->actions[n++] = priv->sh->default_miss.action;
9184 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
9185 (void *)&dv->value, n,
9186 dv->actions, &dh->drv_flow);
9188 rte_flow_error_set(error, errno,
9189 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9191 "hardware refuses to create flow");
9194 if (priv->vmwa_context &&
9195 dh->vf_vlan.tag && !dh->vf_vlan.created) {
9197 * The rule contains the VLAN pattern.
9198 * For VF we are going to create VLAN
9199 * interface to make hypervisor set correct
9200 * e-Switch vport context.
9202 mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
9207 if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9208 flow_dv_default_miss_resource_release(dev);
9210 err = rte_errno; /* Save rte_errno before cleanup. */
9211 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
9212 handle_idx, dh, next) {
9213 /* hrxq is union, don't clear it if the flag is not set. */
9215 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
9216 mlx5_drop_action_destroy(dev);
9218 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
9219 mlx5_hrxq_release(dev, dh->rix_hrxq);
9223 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9224 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9226 rte_errno = err; /* Restore rte_errno. */
9231 * Release the flow matcher.
9234 * Pointer to Ethernet device.
9236 * Pointer to mlx5_flow_handle.
9239 * 1 while a reference on it exists, 0 when freed.
9242 flow_dv_matcher_release(struct rte_eth_dev *dev,
9243 struct mlx5_flow_handle *handle)
9245 struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
9247 MLX5_ASSERT(matcher->matcher_object);
9248 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
9249 dev->data->port_id, (void *)matcher,
9250 rte_atomic32_read(&matcher->refcnt));
9251 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
9252 claim_zero(mlx5_flow_os_destroy_flow_matcher
9253 (matcher->matcher_object));
9254 LIST_REMOVE(matcher, next);
9255 /* table ref-- in release interface. */
9256 flow_dv_tbl_resource_release(dev, matcher->tbl);
9258 DRV_LOG(DEBUG, "port %u matcher %p: removed",
9259 dev->data->port_id, (void *)matcher);
9266 * Release an encap/decap resource.
9269 * Pointer to Ethernet device.
9271 * Pointer to mlx5_flow_handle.
9274 * 1 while a reference on it exists, 0 when freed.
9277 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
9278 struct mlx5_flow_handle *handle)
9280 struct mlx5_priv *priv = dev->data->dev_private;
9281 uint32_t idx = handle->dvh.rix_encap_decap;
9282 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
9284 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
9286 if (!cache_resource)
9288 MLX5_ASSERT(cache_resource->action);
9289 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
9290 (void *)cache_resource,
9291 rte_atomic32_read(&cache_resource->refcnt));
9292 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9293 claim_zero(mlx5_flow_os_destroy_flow_action
9294 (cache_resource->action));
9295 mlx5_hlist_remove(priv->sh->encaps_decaps,
9296 &cache_resource->entry);
9297 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
9298 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
9299 (void *)cache_resource);
9306 * Release an jump to table action resource.
9309 * Pointer to Ethernet device.
9311 * Pointer to mlx5_flow_handle.
9314 * 1 while a reference on it exists, 0 when freed.
9317 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
9318 struct mlx5_flow_handle *handle)
9320 struct mlx5_priv *priv = dev->data->dev_private;
9321 struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
9322 struct mlx5_flow_tbl_data_entry *tbl_data;
9324 tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
9328 cache_resource = &tbl_data->jump;
9329 MLX5_ASSERT(cache_resource->action);
9330 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
9331 (void *)cache_resource,
9332 rte_atomic32_read(&cache_resource->refcnt));
9333 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9334 claim_zero(mlx5_flow_os_destroy_flow_action
9335 (cache_resource->action));
9336 /* jump action memory free is inside the table release. */
9337 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
9338 DRV_LOG(DEBUG, "jump table resource %p: removed",
9339 (void *)cache_resource);
9346 * Release a default miss resource.
9349 * Pointer to Ethernet device.
9351 * 1 while a reference on it exists, 0 when freed.
9354 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
9356 struct mlx5_priv *priv = dev->data->dev_private;
9357 struct mlx5_dev_ctx_shared *sh = priv->sh;
9358 struct mlx5_flow_default_miss_resource *cache_resource =
9361 MLX5_ASSERT(cache_resource->action);
9362 DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
9363 (void *)cache_resource->action,
9364 rte_atomic32_read(&cache_resource->refcnt));
9365 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9366 claim_zero(mlx5_glue->destroy_flow_action
9367 (cache_resource->action));
9368 DRV_LOG(DEBUG, "default miss resource %p: removed",
9369 (void *)cache_resource->action);
9376 * Release a modify-header resource.
9379 * Pointer to Ethernet device.
9381 * Pointer to mlx5_flow_handle.
9384 * 1 while a reference on it exists, 0 when freed.
9387 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
9388 struct mlx5_flow_handle *handle)
9390 struct mlx5_priv *priv = dev->data->dev_private;
9391 struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
9392 handle->dvh.modify_hdr;
9394 MLX5_ASSERT(cache_resource->action);
9395 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
9396 (void *)cache_resource,
9397 rte_atomic32_read(&cache_resource->refcnt));
9398 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9399 claim_zero(mlx5_flow_os_destroy_flow_action
9400 (cache_resource->action));
9401 mlx5_hlist_remove(priv->sh->modify_cmds,
9402 &cache_resource->entry);
9403 mlx5_free(cache_resource);
9404 DRV_LOG(DEBUG, "modify-header resource %p: removed",
9405 (void *)cache_resource);
9412 * Release port ID action resource.
9415 * Pointer to Ethernet device.
9417 * Pointer to mlx5_flow_handle.
9420 * 1 while a reference on it exists, 0 when freed.
9423 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
9424 struct mlx5_flow_handle *handle)
9426 struct mlx5_priv *priv = dev->data->dev_private;
9427 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
9428 uint32_t idx = handle->rix_port_id_action;
9430 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
9432 if (!cache_resource)
9434 MLX5_ASSERT(cache_resource->action);
9435 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
9436 (void *)cache_resource,
9437 rte_atomic32_read(&cache_resource->refcnt));
9438 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9439 claim_zero(mlx5_flow_os_destroy_flow_action
9440 (cache_resource->action));
9441 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
9442 &priv->sh->port_id_action_list, idx,
9443 cache_resource, next);
9444 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
9445 DRV_LOG(DEBUG, "port id action resource %p: removed",
9446 (void *)cache_resource);
9453 * Release push vlan action resource.
9456 * Pointer to Ethernet device.
9458 * Pointer to mlx5_flow_handle.
9461 * 1 while a reference on it exists, 0 when freed.
9464 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
9465 struct mlx5_flow_handle *handle)
9467 struct mlx5_priv *priv = dev->data->dev_private;
9468 uint32_t idx = handle->dvh.rix_push_vlan;
9469 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
9471 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9473 if (!cache_resource)
9475 MLX5_ASSERT(cache_resource->action);
9476 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
9477 (void *)cache_resource,
9478 rte_atomic32_read(&cache_resource->refcnt));
9479 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9480 claim_zero(mlx5_flow_os_destroy_flow_action
9481 (cache_resource->action));
9482 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9483 &priv->sh->push_vlan_action_list, idx,
9484 cache_resource, next);
9485 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
9486 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
9487 (void *)cache_resource);
9494 * Release the fate resource.
9497 * Pointer to Ethernet device.
9499 * Pointer to mlx5_flow_handle.
9502 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
9503 struct mlx5_flow_handle *handle)
9505 if (!handle->rix_fate)
9507 switch (handle->fate_action) {
9508 case MLX5_FLOW_FATE_DROP:
9509 mlx5_drop_action_destroy(dev);
9511 case MLX5_FLOW_FATE_QUEUE:
9512 mlx5_hrxq_release(dev, handle->rix_hrxq);
9514 case MLX5_FLOW_FATE_JUMP:
9515 flow_dv_jump_tbl_resource_release(dev, handle);
9517 case MLX5_FLOW_FATE_PORT_ID:
9518 flow_dv_port_id_action_resource_release(dev, handle);
9520 case MLX5_FLOW_FATE_DEFAULT_MISS:
9521 flow_dv_default_miss_resource_release(dev);
9524 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9527 handle->rix_fate = 0;
9531 * Remove the flow from the NIC but keeps it in memory.
9532 * Lock free, (mutex should be acquired by caller).
9535 * Pointer to Ethernet device.
9536 * @param[in, out] flow
9537 * Pointer to flow structure.
9540 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9542 struct mlx5_flow_handle *dh;
9543 uint32_t handle_idx;
9544 struct mlx5_priv *priv = dev->data->dev_private;
9548 handle_idx = flow->dev_handles;
9549 while (handle_idx) {
9550 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9555 claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
9556 dh->drv_flow = NULL;
9558 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9559 dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9560 dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9561 flow_dv_fate_resource_release(dev, dh);
9562 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9563 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9564 handle_idx = dh->next.next;
9569 * Remove the flow from the NIC and the memory.
9570 * Lock free, (mutex should be acquired by caller).
9573 * Pointer to the Ethernet device structure.
9574 * @param[in, out] flow
9575 * Pointer to flow structure.
9578 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9580 struct mlx5_flow_handle *dev_handle;
9581 struct mlx5_priv *priv = dev->data->dev_private;
9585 __flow_dv_remove(dev, flow);
9586 if (flow->counter) {
9587 flow_dv_counter_release(dev, flow->counter);
9591 struct mlx5_flow_meter *fm;
9593 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9596 mlx5_flow_meter_detach(fm);
9599 while (flow->dev_handles) {
9600 uint32_t tmp_idx = flow->dev_handles;
9602 dev_handle = mlx5_ipool_get(priv->sh->ipool
9603 [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9606 flow->dev_handles = dev_handle->next.next;
9607 if (dev_handle->dvh.matcher)
9608 flow_dv_matcher_release(dev, dev_handle);
9609 if (dev_handle->dvh.rix_encap_decap)
9610 flow_dv_encap_decap_resource_release(dev, dev_handle);
9611 if (dev_handle->dvh.modify_hdr)
9612 flow_dv_modify_hdr_resource_release(dev, dev_handle);
9613 if (dev_handle->dvh.rix_push_vlan)
9614 flow_dv_push_vlan_action_resource_release(dev,
9616 if (dev_handle->dvh.rix_tag)
9617 flow_dv_tag_release(dev,
9618 dev_handle->dvh.rix_tag);
9619 flow_dv_fate_resource_release(dev, dev_handle);
9620 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9626 * Query a dv flow rule for its statistics via devx.
9629 * Pointer to Ethernet device.
9631 * Pointer to the sub flow.
9633 * data retrieved by the query.
9635 * Perform verbose error reporting if not NULL.
9638 * 0 on success, a negative errno value otherwise and rte_errno is set.
9641 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9642 void *data, struct rte_flow_error *error)
9644 struct mlx5_priv *priv = dev->data->dev_private;
9645 struct rte_flow_query_count *qc = data;
9647 if (!priv->config.devx)
9648 return rte_flow_error_set(error, ENOTSUP,
9649 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9651 "counters are not supported");
9652 if (flow->counter) {
9653 uint64_t pkts, bytes;
9654 struct mlx5_flow_counter *cnt;
9656 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9658 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9662 return rte_flow_error_set(error, -err,
9663 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9664 NULL, "cannot read counters");
9667 qc->hits = pkts - cnt->hits;
9668 qc->bytes = bytes - cnt->bytes;
9675 return rte_flow_error_set(error, EINVAL,
9676 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9678 "counters are not available");
9684 * @see rte_flow_query()
9688 flow_dv_query(struct rte_eth_dev *dev,
9689 struct rte_flow *flow __rte_unused,
9690 const struct rte_flow_action *actions __rte_unused,
9691 void *data __rte_unused,
9692 struct rte_flow_error *error __rte_unused)
9696 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9697 switch (actions->type) {
9698 case RTE_FLOW_ACTION_TYPE_VOID:
9700 case RTE_FLOW_ACTION_TYPE_COUNT:
9701 ret = flow_dv_query_count(dev, flow, data, error);
9704 return rte_flow_error_set(error, ENOTSUP,
9705 RTE_FLOW_ERROR_TYPE_ACTION,
9707 "action not supported");
9714 * Destroy the meter table set.
9715 * Lock free, (mutex should be acquired by caller).
9718 * Pointer to Ethernet device.
9720 * Pointer to the meter table set.
9726 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9727 struct mlx5_meter_domains_infos *tbl)
9729 struct mlx5_priv *priv = dev->data->dev_private;
9730 struct mlx5_meter_domains_infos *mtd =
9731 (struct mlx5_meter_domains_infos *)tbl;
9733 if (!mtd || !priv->config.dv_flow_en)
9735 if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9736 claim_zero(mlx5_flow_os_destroy_flow
9737 (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9738 if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9739 claim_zero(mlx5_flow_os_destroy_flow
9740 (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9741 if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9742 claim_zero(mlx5_flow_os_destroy_flow
9743 (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9744 if (mtd->egress.color_matcher)
9745 claim_zero(mlx5_flow_os_destroy_flow_matcher
9746 (mtd->egress.color_matcher));
9747 if (mtd->egress.any_matcher)
9748 claim_zero(mlx5_flow_os_destroy_flow_matcher
9749 (mtd->egress.any_matcher));
9750 if (mtd->egress.tbl)
9751 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9752 if (mtd->egress.sfx_tbl)
9753 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9754 if (mtd->ingress.color_matcher)
9755 claim_zero(mlx5_flow_os_destroy_flow_matcher
9756 (mtd->ingress.color_matcher));
9757 if (mtd->ingress.any_matcher)
9758 claim_zero(mlx5_flow_os_destroy_flow_matcher
9759 (mtd->ingress.any_matcher));
9760 if (mtd->ingress.tbl)
9761 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9762 if (mtd->ingress.sfx_tbl)
9763 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9764 if (mtd->transfer.color_matcher)
9765 claim_zero(mlx5_flow_os_destroy_flow_matcher
9766 (mtd->transfer.color_matcher));
9767 if (mtd->transfer.any_matcher)
9768 claim_zero(mlx5_flow_os_destroy_flow_matcher
9769 (mtd->transfer.any_matcher));
9770 if (mtd->transfer.tbl)
9771 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9772 if (mtd->transfer.sfx_tbl)
9773 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9775 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
9780 /* Number of meter flow actions, count and jump or count and drop. */
9781 #define METER_ACTIONS 2
9784 * Create specify domain meter table and suffix table.
9787 * Pointer to Ethernet device.
9788 * @param[in,out] mtb
9789 * Pointer to DV meter table set.
9792 * @param[in] transfer
9794 * @param[in] color_reg_c_idx
9795 * Reg C index for color match.
9798 * 0 on success, -1 otherwise and rte_errno is set.
9801 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9802 struct mlx5_meter_domains_infos *mtb,
9803 uint8_t egress, uint8_t transfer,
9804 uint32_t color_reg_c_idx)
9806 struct mlx5_priv *priv = dev->data->dev_private;
9807 struct mlx5_dev_ctx_shared *sh = priv->sh;
9808 struct mlx5_flow_dv_match_params mask = {
9809 .size = sizeof(mask.buf),
9811 struct mlx5_flow_dv_match_params value = {
9812 .size = sizeof(value.buf),
9814 struct mlx5dv_flow_matcher_attr dv_attr = {
9815 .type = IBV_FLOW_ATTR_NORMAL,
9817 .match_criteria_enable = 0,
9818 .match_mask = (void *)&mask,
9820 void *actions[METER_ACTIONS];
9821 struct mlx5_meter_domain_info *dtb;
9822 struct rte_flow_error error;
9827 dtb = &mtb->transfer;
9831 dtb = &mtb->ingress;
9832 /* Create the meter table with METER level. */
9833 dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9834 egress, transfer, &error);
9836 DRV_LOG(ERR, "Failed to create meter policer table.");
9839 /* Create the meter suffix table with SUFFIX level. */
9840 dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9841 MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9842 egress, transfer, &error);
9843 if (!dtb->sfx_tbl) {
9844 DRV_LOG(ERR, "Failed to create meter suffix table.");
9847 /* Create matchers, Any and Color. */
9848 dv_attr.priority = 3;
9849 dv_attr.match_criteria_enable = 0;
9850 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9853 DRV_LOG(ERR, "Failed to create meter"
9854 " policer default matcher.");
9857 dv_attr.priority = 0;
9858 dv_attr.match_criteria_enable =
9859 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9860 flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9861 rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9862 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9863 &dtb->color_matcher);
9865 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9868 if (mtb->count_actns[RTE_MTR_DROPPED])
9869 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9870 actions[i++] = mtb->drop_actn;
9871 /* Default rule: lowest priority, match any, actions: drop. */
9872 ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
9874 &dtb->policer_rules[RTE_MTR_DROPPED]);
9876 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9885 * Create the needed meter and suffix tables.
9886 * Lock free, (mutex should be acquired by caller).
9889 * Pointer to Ethernet device.
9891 * Pointer to the flow meter.
9894 * Pointer to table set on success, NULL otherwise and rte_errno is set.
9896 static struct mlx5_meter_domains_infos *
9897 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9898 const struct mlx5_flow_meter *fm)
9900 struct mlx5_priv *priv = dev->data->dev_private;
9901 struct mlx5_meter_domains_infos *mtb;
9905 if (!priv->mtr_en) {
9906 rte_errno = ENOTSUP;
9909 mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
9911 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9914 /* Create meter count actions */
9915 for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9916 struct mlx5_flow_counter *cnt;
9917 if (!fm->policer_stats.cnt[i])
9919 cnt = flow_dv_counter_get_by_idx(dev,
9920 fm->policer_stats.cnt[i], NULL);
9921 mtb->count_actns[i] = cnt->action;
9923 /* Create drop action. */
9924 ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
9926 DRV_LOG(ERR, "Failed to create drop action.");
9929 /* Egress meter table. */
9930 ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9932 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9935 /* Ingress meter table. */
9936 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9938 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9941 /* FDB meter table. */
9942 if (priv->config.dv_esw_en) {
9943 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9944 priv->mtr_color_reg);
9946 DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9952 flow_dv_destroy_mtr_tbl(dev, mtb);
9957 * Destroy domain policer rule.
9960 * Pointer to domain table.
9963 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9967 for (i = 0; i < RTE_MTR_DROPPED; i++) {
9968 if (dt->policer_rules[i]) {
9969 claim_zero(mlx5_flow_os_destroy_flow
9970 (dt->policer_rules[i]));
9971 dt->policer_rules[i] = NULL;
9974 if (dt->jump_actn) {
9975 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
9976 dt->jump_actn = NULL;
9981 * Destroy policer rules.
9984 * Pointer to Ethernet device.
9986 * Pointer to flow meter structure.
9988 * Pointer to flow attributes.
9994 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9995 const struct mlx5_flow_meter *fm,
9996 const struct rte_flow_attr *attr)
9998 struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
10003 flow_dv_destroy_domain_policer_rule(&mtb->egress);
10005 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
10006 if (attr->transfer)
10007 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
10012 * Create specify domain meter policer rule.
10015 * Pointer to flow meter structure.
10017 * Pointer to DV meter table set.
10018 * @param[in] mtr_reg_c
10019 * Color match REG_C.
10022 * 0 on success, -1 otherwise.
10025 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
10026 struct mlx5_meter_domain_info *dtb,
10029 struct mlx5_flow_dv_match_params matcher = {
10030 .size = sizeof(matcher.buf),
10032 struct mlx5_flow_dv_match_params value = {
10033 .size = sizeof(value.buf),
10035 struct mlx5_meter_domains_infos *mtb = fm->mfts;
10036 void *actions[METER_ACTIONS];
10040 /* Create jump action. */
10041 if (!dtb->jump_actn)
10042 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10043 (dtb->sfx_tbl->obj, &dtb->jump_actn);
10045 DRV_LOG(ERR, "Failed to create policer jump action.");
10048 for (i = 0; i < RTE_MTR_DROPPED; i++) {
10051 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
10052 rte_col_2_mlx5_col(i), UINT8_MAX);
10053 if (mtb->count_actns[i])
10054 actions[j++] = mtb->count_actns[i];
10055 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
10056 actions[j++] = mtb->drop_actn;
10058 actions[j++] = dtb->jump_actn;
10059 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
10060 (void *)&value, j, actions,
10061 &dtb->policer_rules[i]);
10063 DRV_LOG(ERR, "Failed to create policer rule.");
10074 * Create policer rules.
10077 * Pointer to Ethernet device.
10079 * Pointer to flow meter structure.
10081 * Pointer to flow attributes.
10084 * 0 on success, -1 otherwise.
10087 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
10088 struct mlx5_flow_meter *fm,
10089 const struct rte_flow_attr *attr)
10091 struct mlx5_priv *priv = dev->data->dev_private;
10092 struct mlx5_meter_domains_infos *mtb = fm->mfts;
10095 if (attr->egress) {
10096 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
10097 priv->mtr_color_reg);
10099 DRV_LOG(ERR, "Failed to create egress policer.");
10103 if (attr->ingress) {
10104 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
10105 priv->mtr_color_reg);
10107 DRV_LOG(ERR, "Failed to create ingress policer.");
10111 if (attr->transfer) {
10112 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
10113 priv->mtr_color_reg);
10115 DRV_LOG(ERR, "Failed to create transfer policer.");
10121 flow_dv_destroy_policer_rules(dev, fm, attr);
10126 * Query a devx counter.
10129 * Pointer to the Ethernet device structure.
10131 * Index to the flow counter.
10133 * Set to clear the counter statistics.
10135 * The statistics value of packets.
10136 * @param[out] bytes
10137 * The statistics value of bytes.
10140 * 0 on success, otherwise return -1.
10143 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
10144 uint64_t *pkts, uint64_t *bytes)
10146 struct mlx5_priv *priv = dev->data->dev_private;
10147 struct mlx5_flow_counter *cnt;
10148 uint64_t inn_pkts, inn_bytes;
10151 if (!priv->config.devx)
10154 ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
10157 cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
10158 *pkts = inn_pkts - cnt->hits;
10159 *bytes = inn_bytes - cnt->bytes;
10161 cnt->hits = inn_pkts;
10162 cnt->bytes = inn_bytes;
10168 * Get aged-out flows.
10171 * Pointer to the Ethernet device structure.
10172 * @param[in] context
10173 * The address of an array of pointers to the aged-out flows contexts.
10174 * @param[in] nb_contexts
10175 * The length of context array pointers.
10176 * @param[out] error
10177 * Perform verbose error reporting if not NULL. Initialized in case of
10181 * how many contexts get in success, otherwise negative errno value.
10182 * if nb_contexts is 0, return the amount of all aged contexts.
10183 * if nb_contexts is not 0 , return the amount of aged flows reported
10184 * in the context array.
10185 * @note: only stub for now
10188 flow_get_aged_flows(struct rte_eth_dev *dev,
10190 uint32_t nb_contexts,
10191 struct rte_flow_error *error)
10193 struct mlx5_priv *priv = dev->data->dev_private;
10194 struct mlx5_age_info *age_info;
10195 struct mlx5_age_param *age_param;
10196 struct mlx5_flow_counter *counter;
10199 if (nb_contexts && !context)
10200 return rte_flow_error_set(error, EINVAL,
10201 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10203 "Should assign at least one flow or"
10204 " context to get if nb_contexts != 0");
10205 age_info = GET_PORT_AGE_INFO(priv);
10206 rte_spinlock_lock(&age_info->aged_sl);
10207 TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
10210 age_param = MLX5_CNT_TO_AGE(counter);
10211 context[nb_flows - 1] = age_param->context;
10212 if (!(--nb_contexts))
10216 rte_spinlock_unlock(&age_info->aged_sl);
10217 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
10222 * Mutex-protected thunk to lock-free __flow_dv_translate().
10225 flow_dv_translate(struct rte_eth_dev *dev,
10226 struct mlx5_flow *dev_flow,
10227 const struct rte_flow_attr *attr,
10228 const struct rte_flow_item items[],
10229 const struct rte_flow_action actions[],
10230 struct rte_flow_error *error)
10234 flow_dv_shared_lock(dev);
10235 ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
10236 flow_dv_shared_unlock(dev);
10241 * Mutex-protected thunk to lock-free __flow_dv_apply().
10244 flow_dv_apply(struct rte_eth_dev *dev,
10245 struct rte_flow *flow,
10246 struct rte_flow_error *error)
10250 flow_dv_shared_lock(dev);
10251 ret = __flow_dv_apply(dev, flow, error);
10252 flow_dv_shared_unlock(dev);
10257 * Mutex-protected thunk to lock-free __flow_dv_remove().
10260 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10262 flow_dv_shared_lock(dev);
10263 __flow_dv_remove(dev, flow);
10264 flow_dv_shared_unlock(dev);
10268 * Mutex-protected thunk to lock-free __flow_dv_destroy().
10271 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10273 flow_dv_shared_lock(dev);
10274 __flow_dv_destroy(dev, flow);
10275 flow_dv_shared_unlock(dev);
10279 * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
10282 flow_dv_counter_allocate(struct rte_eth_dev *dev)
10286 flow_dv_shared_lock(dev);
10287 cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
10288 flow_dv_shared_unlock(dev);
10293 * Mutex-protected thunk to lock-free flow_dv_counter_release().
10296 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
10298 flow_dv_shared_lock(dev);
10299 flow_dv_counter_release(dev, cnt);
10300 flow_dv_shared_unlock(dev);
10303 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
10304 .validate = flow_dv_validate,
10305 .prepare = flow_dv_prepare,
10306 .translate = flow_dv_translate,
10307 .apply = flow_dv_apply,
10308 .remove = flow_dv_remove,
10309 .destroy = flow_dv_destroy,
10310 .query = flow_dv_query,
10311 .create_mtr_tbls = flow_dv_create_mtr_tbl,
10312 .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
10313 .create_policer_rules = flow_dv_create_policer_rules,
10314 .destroy_policer_rules = flow_dv_destroy_policer_rules,
10315 .counter_alloc = flow_dv_counter_allocate,
10316 .counter_free = flow_dv_counter_free,
10317 .counter_query = flow_dv_counter_query,
10318 .get_aged_flows = flow_get_aged_flows,
10321 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */