1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox Technologies, Ltd
6 #include <netinet/in.h>
13 #include <rte_common.h>
14 #include <rte_ether.h>
15 #include <rte_ethdev_driver.h>
17 #include <rte_cycles.h>
18 #include <rte_flow_driver.h>
19 #include <rte_malloc.h>
22 #include <mlx5_glue.h>
23 #include <mlx5_devx_cmds.h>
25 #include <mlx5_malloc.h>
27 #include "mlx5_defs.h"
29 #include "mlx5_flow.h"
30 #include "mlx5_flow_os.h"
31 #include "mlx5_rxtx.h"
33 /** Device flow drivers. */
34 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
36 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
38 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
39 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
40 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
41 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
43 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
44 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
49 MLX5_EXPANSION_ROOT_OUTER,
50 MLX5_EXPANSION_ROOT_ETH_VLAN,
51 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
52 MLX5_EXPANSION_OUTER_ETH,
53 MLX5_EXPANSION_OUTER_ETH_VLAN,
54 MLX5_EXPANSION_OUTER_VLAN,
55 MLX5_EXPANSION_OUTER_IPV4,
56 MLX5_EXPANSION_OUTER_IPV4_UDP,
57 MLX5_EXPANSION_OUTER_IPV4_TCP,
58 MLX5_EXPANSION_OUTER_IPV6,
59 MLX5_EXPANSION_OUTER_IPV6_UDP,
60 MLX5_EXPANSION_OUTER_IPV6_TCP,
62 MLX5_EXPANSION_VXLAN_GPE,
66 MLX5_EXPANSION_ETH_VLAN,
69 MLX5_EXPANSION_IPV4_UDP,
70 MLX5_EXPANSION_IPV4_TCP,
72 MLX5_EXPANSION_IPV6_UDP,
73 MLX5_EXPANSION_IPV6_TCP,
76 /** Supported expansion of items. */
77 static const struct rte_flow_expand_node mlx5_support_expansion[] = {
78 [MLX5_EXPANSION_ROOT] = {
79 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
82 .type = RTE_FLOW_ITEM_TYPE_END,
84 [MLX5_EXPANSION_ROOT_OUTER] = {
85 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
86 MLX5_EXPANSION_OUTER_IPV4,
87 MLX5_EXPANSION_OUTER_IPV6),
88 .type = RTE_FLOW_ITEM_TYPE_END,
90 [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
91 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
92 .type = RTE_FLOW_ITEM_TYPE_END,
94 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
95 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
96 .type = RTE_FLOW_ITEM_TYPE_END,
98 [MLX5_EXPANSION_OUTER_ETH] = {
99 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
100 MLX5_EXPANSION_OUTER_IPV6,
101 MLX5_EXPANSION_MPLS),
102 .type = RTE_FLOW_ITEM_TYPE_ETH,
105 [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
107 .type = RTE_FLOW_ITEM_TYPE_ETH,
110 [MLX5_EXPANSION_OUTER_VLAN] = {
111 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
112 MLX5_EXPANSION_OUTER_IPV6),
113 .type = RTE_FLOW_ITEM_TYPE_VLAN,
115 [MLX5_EXPANSION_OUTER_IPV4] = {
116 .next = RTE_FLOW_EXPAND_RSS_NEXT
117 (MLX5_EXPANSION_OUTER_IPV4_UDP,
118 MLX5_EXPANSION_OUTER_IPV4_TCP,
121 MLX5_EXPANSION_IPV6),
122 .type = RTE_FLOW_ITEM_TYPE_IPV4,
123 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
124 ETH_RSS_NONFRAG_IPV4_OTHER,
126 [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
127 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
128 MLX5_EXPANSION_VXLAN_GPE),
129 .type = RTE_FLOW_ITEM_TYPE_UDP,
130 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
132 [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
133 .type = RTE_FLOW_ITEM_TYPE_TCP,
134 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
136 [MLX5_EXPANSION_OUTER_IPV6] = {
137 .next = RTE_FLOW_EXPAND_RSS_NEXT
138 (MLX5_EXPANSION_OUTER_IPV6_UDP,
139 MLX5_EXPANSION_OUTER_IPV6_TCP,
141 MLX5_EXPANSION_IPV6),
142 .type = RTE_FLOW_ITEM_TYPE_IPV6,
143 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
144 ETH_RSS_NONFRAG_IPV6_OTHER,
146 [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
147 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
148 MLX5_EXPANSION_VXLAN_GPE),
149 .type = RTE_FLOW_ITEM_TYPE_UDP,
150 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
152 [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
153 .type = RTE_FLOW_ITEM_TYPE_TCP,
154 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
156 [MLX5_EXPANSION_VXLAN] = {
157 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
159 MLX5_EXPANSION_IPV6),
160 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
162 [MLX5_EXPANSION_VXLAN_GPE] = {
163 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
165 MLX5_EXPANSION_IPV6),
166 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
168 [MLX5_EXPANSION_GRE] = {
169 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
170 .type = RTE_FLOW_ITEM_TYPE_GRE,
172 [MLX5_EXPANSION_MPLS] = {
173 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
174 MLX5_EXPANSION_IPV6),
175 .type = RTE_FLOW_ITEM_TYPE_MPLS,
177 [MLX5_EXPANSION_ETH] = {
178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
179 MLX5_EXPANSION_IPV6),
180 .type = RTE_FLOW_ITEM_TYPE_ETH,
182 [MLX5_EXPANSION_ETH_VLAN] = {
183 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
184 .type = RTE_FLOW_ITEM_TYPE_ETH,
186 [MLX5_EXPANSION_VLAN] = {
187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
188 MLX5_EXPANSION_IPV6),
189 .type = RTE_FLOW_ITEM_TYPE_VLAN,
191 [MLX5_EXPANSION_IPV4] = {
192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
193 MLX5_EXPANSION_IPV4_TCP),
194 .type = RTE_FLOW_ITEM_TYPE_IPV4,
195 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
196 ETH_RSS_NONFRAG_IPV4_OTHER,
198 [MLX5_EXPANSION_IPV4_UDP] = {
199 .type = RTE_FLOW_ITEM_TYPE_UDP,
200 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
202 [MLX5_EXPANSION_IPV4_TCP] = {
203 .type = RTE_FLOW_ITEM_TYPE_TCP,
204 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
206 [MLX5_EXPANSION_IPV6] = {
207 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
208 MLX5_EXPANSION_IPV6_TCP),
209 .type = RTE_FLOW_ITEM_TYPE_IPV6,
210 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
211 ETH_RSS_NONFRAG_IPV6_OTHER,
213 [MLX5_EXPANSION_IPV6_UDP] = {
214 .type = RTE_FLOW_ITEM_TYPE_UDP,
215 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
217 [MLX5_EXPANSION_IPV6_TCP] = {
218 .type = RTE_FLOW_ITEM_TYPE_TCP,
219 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
223 static const struct rte_flow_ops mlx5_flow_ops = {
224 .validate = mlx5_flow_validate,
225 .create = mlx5_flow_create,
226 .destroy = mlx5_flow_destroy,
227 .flush = mlx5_flow_flush,
228 .isolate = mlx5_flow_isolate,
229 .query = mlx5_flow_query,
230 .dev_dump = mlx5_flow_dev_dump,
231 .get_aged_flows = mlx5_flow_get_aged_flows,
234 /* Convert FDIR request to Generic flow. */
236 struct rte_flow_attr attr;
237 struct rte_flow_item items[4];
238 struct rte_flow_item_eth l2;
239 struct rte_flow_item_eth l2_mask;
241 struct rte_flow_item_ipv4 ipv4;
242 struct rte_flow_item_ipv6 ipv6;
245 struct rte_flow_item_ipv4 ipv4;
246 struct rte_flow_item_ipv6 ipv6;
249 struct rte_flow_item_udp udp;
250 struct rte_flow_item_tcp tcp;
253 struct rte_flow_item_udp udp;
254 struct rte_flow_item_tcp tcp;
256 struct rte_flow_action actions[2];
257 struct rte_flow_action_queue queue;
260 /* Tunnel information. */
261 struct mlx5_flow_tunnel_info {
262 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
263 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
266 static struct mlx5_flow_tunnel_info tunnels_info[] = {
268 .tunnel = MLX5_FLOW_LAYER_VXLAN,
269 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
272 .tunnel = MLX5_FLOW_LAYER_GENEVE,
273 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
276 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
277 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
280 .tunnel = MLX5_FLOW_LAYER_GRE,
281 .ptype = RTE_PTYPE_TUNNEL_GRE,
284 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
285 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
288 .tunnel = MLX5_FLOW_LAYER_MPLS,
289 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
292 .tunnel = MLX5_FLOW_LAYER_NVGRE,
293 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
296 .tunnel = MLX5_FLOW_LAYER_IPIP,
297 .ptype = RTE_PTYPE_TUNNEL_IP,
300 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
301 .ptype = RTE_PTYPE_TUNNEL_IP,
304 .tunnel = MLX5_FLOW_LAYER_GTP,
305 .ptype = RTE_PTYPE_TUNNEL_GTPU,
310 * Translate tag ID to register.
313 * Pointer to the Ethernet device structure.
315 * The feature that request the register.
317 * The request register ID.
319 * Error description in case of any.
322 * The request register on success, a negative errno
323 * value otherwise and rte_errno is set.
326 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
327 enum mlx5_feature_name feature,
329 struct rte_flow_error *error)
331 struct mlx5_priv *priv = dev->data->dev_private;
332 struct mlx5_dev_config *config = &priv->config;
333 enum modify_reg start_reg;
334 bool skip_mtr_reg = false;
337 case MLX5_HAIRPIN_RX:
339 case MLX5_HAIRPIN_TX:
341 case MLX5_METADATA_RX:
342 switch (config->dv_xmeta_en) {
343 case MLX5_XMETA_MODE_LEGACY:
345 case MLX5_XMETA_MODE_META16:
347 case MLX5_XMETA_MODE_META32:
351 case MLX5_METADATA_TX:
353 case MLX5_METADATA_FDB:
354 switch (config->dv_xmeta_en) {
355 case MLX5_XMETA_MODE_LEGACY:
357 case MLX5_XMETA_MODE_META16:
359 case MLX5_XMETA_MODE_META32:
364 switch (config->dv_xmeta_en) {
365 case MLX5_XMETA_MODE_LEGACY:
367 case MLX5_XMETA_MODE_META16:
369 case MLX5_XMETA_MODE_META32:
375 * If meter color and flow match share one register, flow match
376 * should use the meter color register for match.
378 if (priv->mtr_reg_share)
379 return priv->mtr_color_reg;
381 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
384 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE);
385 return priv->mtr_color_reg;
388 * Metadata COPY_MARK register using is in meter suffix sub
389 * flow while with meter. It's safe to share the same register.
391 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
394 * If meter is enable, it will engage the register for color
395 * match and flow match. If meter color match is not using the
396 * REG_C_2, need to skip the REG_C_x be used by meter color
398 * If meter is disable, free to use all available registers.
400 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
401 (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
402 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
403 if (id > (REG_C_7 - start_reg))
404 return rte_flow_error_set(error, EINVAL,
405 RTE_FLOW_ERROR_TYPE_ITEM,
406 NULL, "invalid tag id");
407 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE)
408 return rte_flow_error_set(error, ENOTSUP,
409 RTE_FLOW_ERROR_TYPE_ITEM,
410 NULL, "unsupported tag id");
412 * This case means meter is using the REG_C_x great than 2.
413 * Take care not to conflict with meter color REG_C_x.
414 * If the available index REG_C_y >= REG_C_x, skip the
417 if (skip_mtr_reg && config->flow_mreg_c
418 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
419 if (id >= (REG_C_7 - start_reg))
420 return rte_flow_error_set(error, EINVAL,
421 RTE_FLOW_ERROR_TYPE_ITEM,
422 NULL, "invalid tag id");
423 if (config->flow_mreg_c
424 [id + 1 + start_reg - REG_C_0] != REG_NONE)
425 return config->flow_mreg_c
426 [id + 1 + start_reg - REG_C_0];
427 return rte_flow_error_set(error, ENOTSUP,
428 RTE_FLOW_ERROR_TYPE_ITEM,
429 NULL, "unsupported tag id");
431 return config->flow_mreg_c[id + start_reg - REG_C_0];
434 return rte_flow_error_set(error, EINVAL,
435 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
436 NULL, "invalid feature name");
440 * Check extensive flow metadata register support.
443 * Pointer to rte_eth_dev structure.
446 * True if device supports extensive flow metadata register, otherwise false.
449 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
451 struct mlx5_priv *priv = dev->data->dev_private;
452 struct mlx5_dev_config *config = &priv->config;
455 * Having available reg_c can be regarded inclusively as supporting
456 * extensive flow metadata register, which could mean,
457 * - metadata register copy action by modify header.
458 * - 16 modify header actions is supported.
459 * - reg_c's are preserved across different domain (FDB and NIC) on
460 * packet loopback by flow lookup miss.
462 return config->flow_mreg_c[2] != REG_NONE;
466 * Verify the @p item specifications (spec, last, mask) are compatible with the
470 * Item specification.
472 * @p item->mask or flow default bit-masks.
473 * @param[in] nic_mask
474 * Bit-masks covering supported fields by the NIC to compare with user mask.
476 * Bit-masks size in bytes.
478 * Pointer to error structure.
481 * 0 on success, a negative errno value otherwise and rte_errno is set.
484 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
486 const uint8_t *nic_mask,
488 struct rte_flow_error *error)
492 MLX5_ASSERT(nic_mask);
493 for (i = 0; i < size; ++i)
494 if ((nic_mask[i] | mask[i]) != nic_mask[i])
495 return rte_flow_error_set(error, ENOTSUP,
496 RTE_FLOW_ERROR_TYPE_ITEM,
498 "mask enables non supported"
500 if (!item->spec && (item->mask || item->last))
501 return rte_flow_error_set(error, EINVAL,
502 RTE_FLOW_ERROR_TYPE_ITEM, item,
503 "mask/last without a spec is not"
505 if (item->spec && item->last) {
511 for (i = 0; i < size; ++i) {
512 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
513 last[i] = ((const uint8_t *)item->last)[i] & mask[i];
515 ret = memcmp(spec, last, size);
517 return rte_flow_error_set(error, EINVAL,
518 RTE_FLOW_ERROR_TYPE_ITEM,
520 "range is not valid");
526 * Adjust the hash fields according to the @p flow information.
528 * @param[in] dev_flow.
529 * Pointer to the mlx5_flow.
531 * 1 when the hash field is for a tunnel item.
532 * @param[in] layer_types
534 * @param[in] hash_fields
538 * The hash fields that should be used.
541 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
542 int tunnel __rte_unused, uint64_t layer_types,
543 uint64_t hash_fields)
545 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
546 int rss_request_inner = rss_desc->level >= 2;
548 /* Check RSS hash level for tunnel. */
549 if (tunnel && rss_request_inner)
550 hash_fields |= IBV_RX_HASH_INNER;
551 else if (tunnel || rss_request_inner)
554 /* Check if requested layer matches RSS hash fields. */
555 if (!(rss_desc->types & layer_types))
561 * Lookup and set the ptype in the data Rx part. A single Ptype can be used,
562 * if several tunnel rules are used on this queue, the tunnel ptype will be
566 * Rx queue to update.
569 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
572 uint32_t tunnel_ptype = 0;
574 /* Look up for the ptype to use. */
575 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
576 if (!rxq_ctrl->flow_tunnels_n[i])
579 tunnel_ptype = tunnels_info[i].ptype;
585 rxq_ctrl->rxq.tunnel = tunnel_ptype;
589 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
593 * Pointer to the Ethernet device structure.
594 * @param[in] dev_handle
595 * Pointer to device flow handle structure.
598 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
599 struct mlx5_flow_handle *dev_handle)
601 struct mlx5_priv *priv = dev->data->dev_private;
602 const int mark = dev_handle->mark;
603 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
604 struct mlx5_hrxq *hrxq;
607 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
609 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
610 dev_handle->rix_hrxq);
613 for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
614 int idx = hrxq->ind_table->queues[i];
615 struct mlx5_rxq_ctrl *rxq_ctrl =
616 container_of((*priv->rxqs)[idx],
617 struct mlx5_rxq_ctrl, rxq);
620 * To support metadata register copy on Tx loopback,
621 * this must be always enabled (metadata may arive
622 * from other port - not from local flows only.
624 if (priv->config.dv_flow_en &&
625 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
626 mlx5_flow_ext_mreg_supported(dev)) {
627 rxq_ctrl->rxq.mark = 1;
628 rxq_ctrl->flow_mark_n = 1;
630 rxq_ctrl->rxq.mark = 1;
631 rxq_ctrl->flow_mark_n++;
636 /* Increase the counter matching the flow. */
637 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
638 if ((tunnels_info[j].tunnel &
639 dev_handle->layers) ==
640 tunnels_info[j].tunnel) {
641 rxq_ctrl->flow_tunnels_n[j]++;
645 flow_rxq_tunnel_ptype_update(rxq_ctrl);
651 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
654 * Pointer to the Ethernet device structure.
656 * Pointer to flow structure.
659 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
661 struct mlx5_priv *priv = dev->data->dev_private;
663 struct mlx5_flow_handle *dev_handle;
665 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
666 handle_idx, dev_handle, next)
667 flow_drv_rxq_flags_set(dev, dev_handle);
671 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
672 * device flow if no other flow uses it with the same kind of request.
675 * Pointer to Ethernet device.
676 * @param[in] dev_handle
677 * Pointer to the device flow handle structure.
680 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
681 struct mlx5_flow_handle *dev_handle)
683 struct mlx5_priv *priv = dev->data->dev_private;
684 const int mark = dev_handle->mark;
685 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
686 struct mlx5_hrxq *hrxq;
689 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
691 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
692 dev_handle->rix_hrxq);
695 MLX5_ASSERT(dev->data->dev_started);
696 for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
697 int idx = hrxq->ind_table->queues[i];
698 struct mlx5_rxq_ctrl *rxq_ctrl =
699 container_of((*priv->rxqs)[idx],
700 struct mlx5_rxq_ctrl, rxq);
702 if (priv->config.dv_flow_en &&
703 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
704 mlx5_flow_ext_mreg_supported(dev)) {
705 rxq_ctrl->rxq.mark = 1;
706 rxq_ctrl->flow_mark_n = 1;
708 rxq_ctrl->flow_mark_n--;
709 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
714 /* Decrease the counter matching the flow. */
715 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
716 if ((tunnels_info[j].tunnel &
717 dev_handle->layers) ==
718 tunnels_info[j].tunnel) {
719 rxq_ctrl->flow_tunnels_n[j]--;
723 flow_rxq_tunnel_ptype_update(rxq_ctrl);
729 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
730 * @p flow if no other flow uses it with the same kind of request.
733 * Pointer to Ethernet device.
735 * Pointer to the flow.
738 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
740 struct mlx5_priv *priv = dev->data->dev_private;
742 struct mlx5_flow_handle *dev_handle;
744 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
745 handle_idx, dev_handle, next)
746 flow_drv_rxq_flags_trim(dev, dev_handle);
750 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
753 * Pointer to Ethernet device.
756 flow_rxq_flags_clear(struct rte_eth_dev *dev)
758 struct mlx5_priv *priv = dev->data->dev_private;
761 for (i = 0; i != priv->rxqs_n; ++i) {
762 struct mlx5_rxq_ctrl *rxq_ctrl;
765 if (!(*priv->rxqs)[i])
767 rxq_ctrl = container_of((*priv->rxqs)[i],
768 struct mlx5_rxq_ctrl, rxq);
769 rxq_ctrl->flow_mark_n = 0;
770 rxq_ctrl->rxq.mark = 0;
771 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
772 rxq_ctrl->flow_tunnels_n[j] = 0;
773 rxq_ctrl->rxq.tunnel = 0;
778 * Set the Rx queue dynamic metadata (mask and offset) for a flow
781 * Pointer to the Ethernet device structure.
784 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
786 struct mlx5_priv *priv = dev->data->dev_private;
787 struct mlx5_rxq_data *data;
790 for (i = 0; i != priv->rxqs_n; ++i) {
791 if (!(*priv->rxqs)[i])
793 data = (*priv->rxqs)[i];
794 if (!rte_flow_dynf_metadata_avail()) {
796 data->flow_meta_mask = 0;
797 data->flow_meta_offset = -1;
800 data->flow_meta_mask = rte_flow_dynf_metadata_mask;
801 data->flow_meta_offset = rte_flow_dynf_metadata_offs;
807 * return a pointer to the desired action in the list of actions.
810 * The list of actions to search the action in.
812 * The action to find.
815 * Pointer to the action in the list, if found. NULL otherwise.
817 const struct rte_flow_action *
818 mlx5_flow_find_action(const struct rte_flow_action *actions,
819 enum rte_flow_action_type action)
823 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
824 if (actions->type == action)
830 * Validate the flag action.
832 * @param[in] action_flags
833 * Bit-fields that holds the actions detected until now.
835 * Attributes of flow that includes this action.
837 * Pointer to error structure.
840 * 0 on success, a negative errno value otherwise and rte_errno is set.
843 mlx5_flow_validate_action_flag(uint64_t action_flags,
844 const struct rte_flow_attr *attr,
845 struct rte_flow_error *error)
847 if (action_flags & MLX5_FLOW_ACTION_MARK)
848 return rte_flow_error_set(error, EINVAL,
849 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
850 "can't mark and flag in same flow");
851 if (action_flags & MLX5_FLOW_ACTION_FLAG)
852 return rte_flow_error_set(error, EINVAL,
853 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
855 " actions in same flow");
857 return rte_flow_error_set(error, ENOTSUP,
858 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
859 "flag action not supported for "
865 * Validate the mark action.
868 * Pointer to the queue action.
869 * @param[in] action_flags
870 * Bit-fields that holds the actions detected until now.
872 * Attributes of flow that includes this action.
874 * Pointer to error structure.
877 * 0 on success, a negative errno value otherwise and rte_errno is set.
880 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
881 uint64_t action_flags,
882 const struct rte_flow_attr *attr,
883 struct rte_flow_error *error)
885 const struct rte_flow_action_mark *mark = action->conf;
888 return rte_flow_error_set(error, EINVAL,
889 RTE_FLOW_ERROR_TYPE_ACTION,
891 "configuration cannot be null");
892 if (mark->id >= MLX5_FLOW_MARK_MAX)
893 return rte_flow_error_set(error, EINVAL,
894 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
896 "mark id must in 0 <= id < "
897 RTE_STR(MLX5_FLOW_MARK_MAX));
898 if (action_flags & MLX5_FLOW_ACTION_FLAG)
899 return rte_flow_error_set(error, EINVAL,
900 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
901 "can't flag and mark in same flow");
902 if (action_flags & MLX5_FLOW_ACTION_MARK)
903 return rte_flow_error_set(error, EINVAL,
904 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
905 "can't have 2 mark actions in same"
908 return rte_flow_error_set(error, ENOTSUP,
909 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
910 "mark action not supported for "
916 * Validate the drop action.
918 * @param[in] action_flags
919 * Bit-fields that holds the actions detected until now.
921 * Attributes of flow that includes this action.
923 * Pointer to error structure.
926 * 0 on success, a negative errno value otherwise and rte_errno is set.
929 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
930 const struct rte_flow_attr *attr,
931 struct rte_flow_error *error)
934 return rte_flow_error_set(error, ENOTSUP,
935 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
936 "drop action not supported for "
942 * Validate the queue action.
945 * Pointer to the queue action.
946 * @param[in] action_flags
947 * Bit-fields that holds the actions detected until now.
949 * Pointer to the Ethernet device structure.
951 * Attributes of flow that includes this action.
953 * Pointer to error structure.
956 * 0 on success, a negative errno value otherwise and rte_errno is set.
959 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
960 uint64_t action_flags,
961 struct rte_eth_dev *dev,
962 const struct rte_flow_attr *attr,
963 struct rte_flow_error *error)
965 struct mlx5_priv *priv = dev->data->dev_private;
966 const struct rte_flow_action_queue *queue = action->conf;
968 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
969 return rte_flow_error_set(error, EINVAL,
970 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
971 "can't have 2 fate actions in"
974 return rte_flow_error_set(error, EINVAL,
975 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
976 NULL, "No Rx queues configured");
977 if (queue->index >= priv->rxqs_n)
978 return rte_flow_error_set(error, EINVAL,
979 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
981 "queue index out of range");
982 if (!(*priv->rxqs)[queue->index])
983 return rte_flow_error_set(error, EINVAL,
984 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
986 "queue is not configured");
988 return rte_flow_error_set(error, ENOTSUP,
989 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
990 "queue action not supported for "
996 * Validate the rss action.
999 * Pointer to the queue action.
1000 * @param[in] action_flags
1001 * Bit-fields that holds the actions detected until now.
1003 * Pointer to the Ethernet device structure.
1005 * Attributes of flow that includes this action.
1006 * @param[in] item_flags
1007 * Items that were detected.
1009 * Pointer to error structure.
1012 * 0 on success, a negative errno value otherwise and rte_errno is set.
1015 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1016 uint64_t action_flags,
1017 struct rte_eth_dev *dev,
1018 const struct rte_flow_attr *attr,
1019 uint64_t item_flags,
1020 struct rte_flow_error *error)
1022 struct mlx5_priv *priv = dev->data->dev_private;
1023 const struct rte_flow_action_rss *rss = action->conf;
1024 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1027 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1028 return rte_flow_error_set(error, EINVAL,
1029 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1030 "can't have 2 fate actions"
1032 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1033 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1034 return rte_flow_error_set(error, ENOTSUP,
1035 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1037 "RSS hash function not supported");
1038 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1043 return rte_flow_error_set(error, ENOTSUP,
1044 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1046 "tunnel RSS is not supported");
1047 /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1048 if (rss->key_len == 0 && rss->key != NULL)
1049 return rte_flow_error_set(error, ENOTSUP,
1050 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1052 "RSS hash key length 0");
1053 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1054 return rte_flow_error_set(error, ENOTSUP,
1055 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1057 "RSS hash key too small");
1058 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1059 return rte_flow_error_set(error, ENOTSUP,
1060 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1062 "RSS hash key too large");
1063 if (rss->queue_num > priv->config.ind_table_max_size)
1064 return rte_flow_error_set(error, ENOTSUP,
1065 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1067 "number of queues too large");
1068 if (rss->types & MLX5_RSS_HF_MASK)
1069 return rte_flow_error_set(error, ENOTSUP,
1070 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1072 "some RSS protocols are not"
1074 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1075 !(rss->types & ETH_RSS_IP))
1076 return rte_flow_error_set(error, EINVAL,
1077 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1078 "L3 partial RSS requested but L3 RSS"
1079 " type not specified");
1080 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1081 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1082 return rte_flow_error_set(error, EINVAL,
1083 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1084 "L4 partial RSS requested but L4 RSS"
1085 " type not specified");
1087 return rte_flow_error_set(error, EINVAL,
1088 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1089 NULL, "No Rx queues configured");
1090 if (!rss->queue_num)
1091 return rte_flow_error_set(error, EINVAL,
1092 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1093 NULL, "No queues configured");
1094 for (i = 0; i != rss->queue_num; ++i) {
1095 if (rss->queue[i] >= priv->rxqs_n)
1096 return rte_flow_error_set
1098 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1099 &rss->queue[i], "queue index out of range");
1100 if (!(*priv->rxqs)[rss->queue[i]])
1101 return rte_flow_error_set
1102 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1103 &rss->queue[i], "queue is not configured");
1106 return rte_flow_error_set(error, ENOTSUP,
1107 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1108 "rss action not supported for "
1110 if (rss->level > 1 && !tunnel)
1111 return rte_flow_error_set(error, EINVAL,
1112 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1113 "inner RSS is not supported for "
1114 "non-tunnel flows");
1115 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1116 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1117 return rte_flow_error_set(error, EINVAL,
1118 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1119 "RSS on eCPRI is not supported now");
1125 * Validate the default miss action.
1127 * @param[in] action_flags
1128 * Bit-fields that holds the actions detected until now.
1130 * Pointer to error structure.
1133 * 0 on success, a negative errno value otherwise and rte_errno is set.
1136 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1137 const struct rte_flow_attr *attr,
1138 struct rte_flow_error *error)
1140 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1141 return rte_flow_error_set(error, EINVAL,
1142 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1143 "can't have 2 fate actions in"
1146 return rte_flow_error_set(error, ENOTSUP,
1147 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1148 "default miss action not supported "
1151 return rte_flow_error_set(error, ENOTSUP,
1152 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1153 "only group 0 is supported");
1155 return rte_flow_error_set(error, ENOTSUP,
1156 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1157 NULL, "transfer is not supported");
1162 * Validate the count action.
1165 * Pointer to the Ethernet device structure.
1167 * Attributes of flow that includes this action.
1169 * Pointer to error structure.
1172 * 0 on success, a negative errno value otherwise and rte_errno is set.
1175 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1176 const struct rte_flow_attr *attr,
1177 struct rte_flow_error *error)
1180 return rte_flow_error_set(error, ENOTSUP,
1181 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1182 "count action not supported for "
1188 * Verify the @p attributes will be correctly understood by the NIC and store
1189 * them in the @p flow if everything is correct.
1192 * Pointer to the Ethernet device structure.
1193 * @param[in] attributes
1194 * Pointer to flow attributes
1196 * Pointer to error structure.
1199 * 0 on success, a negative errno value otherwise and rte_errno is set.
1202 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1203 const struct rte_flow_attr *attributes,
1204 struct rte_flow_error *error)
1206 struct mlx5_priv *priv = dev->data->dev_private;
1207 uint32_t priority_max = priv->config.flow_prio - 1;
1209 if (attributes->group)
1210 return rte_flow_error_set(error, ENOTSUP,
1211 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1212 NULL, "groups is not supported");
1213 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1214 attributes->priority >= priority_max)
1215 return rte_flow_error_set(error, ENOTSUP,
1216 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1217 NULL, "priority out of range");
1218 if (attributes->egress)
1219 return rte_flow_error_set(error, ENOTSUP,
1220 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1221 "egress is not supported");
1222 if (attributes->transfer && !priv->config.dv_esw_en)
1223 return rte_flow_error_set(error, ENOTSUP,
1224 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1225 NULL, "transfer is not supported");
1226 if (!attributes->ingress)
1227 return rte_flow_error_set(error, EINVAL,
1228 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1230 "ingress attribute is mandatory");
1235 * Validate ICMP6 item.
1238 * Item specification.
1239 * @param[in] item_flags
1240 * Bit-fields that holds the items detected until now.
1242 * Pointer to error structure.
1245 * 0 on success, a negative errno value otherwise and rte_errno is set.
1248 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1249 uint64_t item_flags,
1250 uint8_t target_protocol,
1251 struct rte_flow_error *error)
1253 const struct rte_flow_item_icmp6 *mask = item->mask;
1254 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1255 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1256 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1257 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1258 MLX5_FLOW_LAYER_OUTER_L4;
1261 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1262 return rte_flow_error_set(error, EINVAL,
1263 RTE_FLOW_ERROR_TYPE_ITEM, item,
1264 "protocol filtering not compatible"
1265 " with ICMP6 layer");
1266 if (!(item_flags & l3m))
1267 return rte_flow_error_set(error, EINVAL,
1268 RTE_FLOW_ERROR_TYPE_ITEM, item,
1269 "IPv6 is mandatory to filter on"
1271 if (item_flags & l4m)
1272 return rte_flow_error_set(error, EINVAL,
1273 RTE_FLOW_ERROR_TYPE_ITEM, item,
1274 "multiple L4 layers not supported");
1276 mask = &rte_flow_item_icmp6_mask;
1277 ret = mlx5_flow_item_acceptable
1278 (item, (const uint8_t *)mask,
1279 (const uint8_t *)&rte_flow_item_icmp6_mask,
1280 sizeof(struct rte_flow_item_icmp6), error);
1287 * Validate ICMP item.
1290 * Item specification.
1291 * @param[in] item_flags
1292 * Bit-fields that holds the items detected until now.
1294 * Pointer to error structure.
1297 * 0 on success, a negative errno value otherwise and rte_errno is set.
1300 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1301 uint64_t item_flags,
1302 uint8_t target_protocol,
1303 struct rte_flow_error *error)
1305 const struct rte_flow_item_icmp *mask = item->mask;
1306 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1307 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1308 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1309 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1310 MLX5_FLOW_LAYER_OUTER_L4;
1313 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1314 return rte_flow_error_set(error, EINVAL,
1315 RTE_FLOW_ERROR_TYPE_ITEM, item,
1316 "protocol filtering not compatible"
1317 " with ICMP layer");
1318 if (!(item_flags & l3m))
1319 return rte_flow_error_set(error, EINVAL,
1320 RTE_FLOW_ERROR_TYPE_ITEM, item,
1321 "IPv4 is mandatory to filter"
1323 if (item_flags & l4m)
1324 return rte_flow_error_set(error, EINVAL,
1325 RTE_FLOW_ERROR_TYPE_ITEM, item,
1326 "multiple L4 layers not supported");
1328 mask = &rte_flow_item_icmp_mask;
1329 ret = mlx5_flow_item_acceptable
1330 (item, (const uint8_t *)mask,
1331 (const uint8_t *)&rte_flow_item_icmp_mask,
1332 sizeof(struct rte_flow_item_icmp), error);
1339 * Validate Ethernet item.
1342 * Item specification.
1343 * @param[in] item_flags
1344 * Bit-fields that holds the items detected until now.
1346 * Pointer to error structure.
1349 * 0 on success, a negative errno value otherwise and rte_errno is set.
1352 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1353 uint64_t item_flags,
1354 struct rte_flow_error *error)
1356 const struct rte_flow_item_eth *mask = item->mask;
1357 const struct rte_flow_item_eth nic_mask = {
1358 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1359 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1360 .type = RTE_BE16(0xffff),
1363 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1364 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1365 MLX5_FLOW_LAYER_OUTER_L2;
1367 if (item_flags & ethm)
1368 return rte_flow_error_set(error, ENOTSUP,
1369 RTE_FLOW_ERROR_TYPE_ITEM, item,
1370 "multiple L2 layers not supported");
1371 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1372 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1373 return rte_flow_error_set(error, EINVAL,
1374 RTE_FLOW_ERROR_TYPE_ITEM, item,
1375 "L2 layer should not follow "
1377 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1378 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1379 return rte_flow_error_set(error, EINVAL,
1380 RTE_FLOW_ERROR_TYPE_ITEM, item,
1381 "L2 layer should not follow VLAN");
1383 mask = &rte_flow_item_eth_mask;
1384 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1385 (const uint8_t *)&nic_mask,
1386 sizeof(struct rte_flow_item_eth),
1392 * Validate VLAN item.
1395 * Item specification.
1396 * @param[in] item_flags
1397 * Bit-fields that holds the items detected until now.
1399 * Ethernet device flow is being created on.
1401 * Pointer to error structure.
1404 * 0 on success, a negative errno value otherwise and rte_errno is set.
1407 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1408 uint64_t item_flags,
1409 struct rte_eth_dev *dev,
1410 struct rte_flow_error *error)
1412 const struct rte_flow_item_vlan *spec = item->spec;
1413 const struct rte_flow_item_vlan *mask = item->mask;
1414 const struct rte_flow_item_vlan nic_mask = {
1415 .tci = RTE_BE16(UINT16_MAX),
1416 .inner_type = RTE_BE16(UINT16_MAX),
1418 uint16_t vlan_tag = 0;
1419 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1421 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1422 MLX5_FLOW_LAYER_INNER_L4) :
1423 (MLX5_FLOW_LAYER_OUTER_L3 |
1424 MLX5_FLOW_LAYER_OUTER_L4);
1425 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1426 MLX5_FLOW_LAYER_OUTER_VLAN;
1428 if (item_flags & vlanm)
1429 return rte_flow_error_set(error, EINVAL,
1430 RTE_FLOW_ERROR_TYPE_ITEM, item,
1431 "multiple VLAN layers not supported");
1432 else if ((item_flags & l34m) != 0)
1433 return rte_flow_error_set(error, EINVAL,
1434 RTE_FLOW_ERROR_TYPE_ITEM, item,
1435 "VLAN cannot follow L3/L4 layer");
1437 mask = &rte_flow_item_vlan_mask;
1438 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1439 (const uint8_t *)&nic_mask,
1440 sizeof(struct rte_flow_item_vlan),
1444 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1445 struct mlx5_priv *priv = dev->data->dev_private;
1447 if (priv->vmwa_context) {
1449 * Non-NULL context means we have a virtual machine
1450 * and SR-IOV enabled, we have to create VLAN interface
1451 * to make hypervisor to setup E-Switch vport
1452 * context correctly. We avoid creating the multiple
1453 * VLAN interfaces, so we cannot support VLAN tag mask.
1455 return rte_flow_error_set(error, EINVAL,
1456 RTE_FLOW_ERROR_TYPE_ITEM,
1458 "VLAN tag mask is not"
1459 " supported in virtual"
1464 vlan_tag = spec->tci;
1465 vlan_tag &= mask->tci;
1468 * From verbs perspective an empty VLAN is equivalent
1469 * to a packet without VLAN layer.
1472 return rte_flow_error_set(error, EINVAL,
1473 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1475 "VLAN cannot be empty");
1480 * Validate IPV4 item.
1483 * Item specification.
1484 * @param[in] item_flags
1485 * Bit-fields that holds the items detected until now.
1486 * @param[in] last_item
1487 * Previous validated item in the pattern items.
1488 * @param[in] ether_type
1489 * Type in the ethernet layer header (including dot1q).
1490 * @param[in] acc_mask
1491 * Acceptable mask, if NULL default internal default mask
1492 * will be used to check whether item fields are supported.
1494 * Pointer to error structure.
1497 * 0 on success, a negative errno value otherwise and rte_errno is set.
1500 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1501 uint64_t item_flags,
1503 uint16_t ether_type,
1504 const struct rte_flow_item_ipv4 *acc_mask,
1505 struct rte_flow_error *error)
1507 const struct rte_flow_item_ipv4 *mask = item->mask;
1508 const struct rte_flow_item_ipv4 *spec = item->spec;
1509 const struct rte_flow_item_ipv4 nic_mask = {
1511 .src_addr = RTE_BE32(0xffffffff),
1512 .dst_addr = RTE_BE32(0xffffffff),
1513 .type_of_service = 0xff,
1514 .next_proto_id = 0xff,
1517 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1518 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1519 MLX5_FLOW_LAYER_OUTER_L3;
1520 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1521 MLX5_FLOW_LAYER_OUTER_L4;
1523 uint8_t next_proto = 0xFF;
1524 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1525 MLX5_FLOW_LAYER_OUTER_VLAN |
1526 MLX5_FLOW_LAYER_INNER_VLAN);
1528 if ((last_item & l2_vlan) && ether_type &&
1529 ether_type != RTE_ETHER_TYPE_IPV4)
1530 return rte_flow_error_set(error, EINVAL,
1531 RTE_FLOW_ERROR_TYPE_ITEM, item,
1532 "IPv4 cannot follow L2/VLAN layer "
1533 "which ether type is not IPv4");
1534 if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1536 next_proto = mask->hdr.next_proto_id &
1537 spec->hdr.next_proto_id;
1538 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1539 return rte_flow_error_set(error, EINVAL,
1540 RTE_FLOW_ERROR_TYPE_ITEM,
1545 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1546 return rte_flow_error_set(error, EINVAL,
1547 RTE_FLOW_ERROR_TYPE_ITEM, item,
1548 "wrong tunnel type - IPv6 specified "
1549 "but IPv4 item provided");
1550 if (item_flags & l3m)
1551 return rte_flow_error_set(error, ENOTSUP,
1552 RTE_FLOW_ERROR_TYPE_ITEM, item,
1553 "multiple L3 layers not supported");
1554 else if (item_flags & l4m)
1555 return rte_flow_error_set(error, EINVAL,
1556 RTE_FLOW_ERROR_TYPE_ITEM, item,
1557 "L3 cannot follow an L4 layer.");
1558 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1559 !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1560 return rte_flow_error_set(error, EINVAL,
1561 RTE_FLOW_ERROR_TYPE_ITEM, item,
1562 "L3 cannot follow an NVGRE layer.");
1564 mask = &rte_flow_item_ipv4_mask;
1565 else if (mask->hdr.next_proto_id != 0 &&
1566 mask->hdr.next_proto_id != 0xff)
1567 return rte_flow_error_set(error, EINVAL,
1568 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1569 "partial mask is not supported"
1571 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1572 acc_mask ? (const uint8_t *)acc_mask
1573 : (const uint8_t *)&nic_mask,
1574 sizeof(struct rte_flow_item_ipv4),
1582 * Validate IPV6 item.
1585 * Item specification.
1586 * @param[in] item_flags
1587 * Bit-fields that holds the items detected until now.
1588 * @param[in] last_item
1589 * Previous validated item in the pattern items.
1590 * @param[in] ether_type
1591 * Type in the ethernet layer header (including dot1q).
1592 * @param[in] acc_mask
1593 * Acceptable mask, if NULL default internal default mask
1594 * will be used to check whether item fields are supported.
1596 * Pointer to error structure.
1599 * 0 on success, a negative errno value otherwise and rte_errno is set.
1602 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1603 uint64_t item_flags,
1605 uint16_t ether_type,
1606 const struct rte_flow_item_ipv6 *acc_mask,
1607 struct rte_flow_error *error)
1609 const struct rte_flow_item_ipv6 *mask = item->mask;
1610 const struct rte_flow_item_ipv6 *spec = item->spec;
1611 const struct rte_flow_item_ipv6 nic_mask = {
1614 "\xff\xff\xff\xff\xff\xff\xff\xff"
1615 "\xff\xff\xff\xff\xff\xff\xff\xff",
1617 "\xff\xff\xff\xff\xff\xff\xff\xff"
1618 "\xff\xff\xff\xff\xff\xff\xff\xff",
1619 .vtc_flow = RTE_BE32(0xffffffff),
1623 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1624 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1625 MLX5_FLOW_LAYER_OUTER_L3;
1626 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1627 MLX5_FLOW_LAYER_OUTER_L4;
1629 uint8_t next_proto = 0xFF;
1630 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1631 MLX5_FLOW_LAYER_OUTER_VLAN |
1632 MLX5_FLOW_LAYER_INNER_VLAN);
1634 if ((last_item & l2_vlan) && ether_type &&
1635 ether_type != RTE_ETHER_TYPE_IPV6)
1636 return rte_flow_error_set(error, EINVAL,
1637 RTE_FLOW_ERROR_TYPE_ITEM, item,
1638 "IPv6 cannot follow L2/VLAN layer "
1639 "which ether type is not IPv6");
1640 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
1642 next_proto = mask->hdr.proto & spec->hdr.proto;
1643 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1644 return rte_flow_error_set(error, EINVAL,
1645 RTE_FLOW_ERROR_TYPE_ITEM,
1650 if (item_flags & MLX5_FLOW_LAYER_IPIP)
1651 return rte_flow_error_set(error, EINVAL,
1652 RTE_FLOW_ERROR_TYPE_ITEM, item,
1653 "wrong tunnel type - IPv4 specified "
1654 "but IPv6 item provided");
1655 if (item_flags & l3m)
1656 return rte_flow_error_set(error, ENOTSUP,
1657 RTE_FLOW_ERROR_TYPE_ITEM, item,
1658 "multiple L3 layers not supported");
1659 else if (item_flags & l4m)
1660 return rte_flow_error_set(error, EINVAL,
1661 RTE_FLOW_ERROR_TYPE_ITEM, item,
1662 "L3 cannot follow an L4 layer.");
1663 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1664 !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1665 return rte_flow_error_set(error, EINVAL,
1666 RTE_FLOW_ERROR_TYPE_ITEM, item,
1667 "L3 cannot follow an NVGRE layer.");
1669 mask = &rte_flow_item_ipv6_mask;
1670 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1671 acc_mask ? (const uint8_t *)acc_mask
1672 : (const uint8_t *)&nic_mask,
1673 sizeof(struct rte_flow_item_ipv6),
1681 * Validate UDP item.
1684 * Item specification.
1685 * @param[in] item_flags
1686 * Bit-fields that holds the items detected until now.
1687 * @param[in] target_protocol
1688 * The next protocol in the previous item.
1689 * @param[in] flow_mask
1690 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
1692 * Pointer to error structure.
1695 * 0 on success, a negative errno value otherwise and rte_errno is set.
1698 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
1699 uint64_t item_flags,
1700 uint8_t target_protocol,
1701 struct rte_flow_error *error)
1703 const struct rte_flow_item_udp *mask = item->mask;
1704 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1705 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1706 MLX5_FLOW_LAYER_OUTER_L3;
1707 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1708 MLX5_FLOW_LAYER_OUTER_L4;
1711 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
1712 return rte_flow_error_set(error, EINVAL,
1713 RTE_FLOW_ERROR_TYPE_ITEM, item,
1714 "protocol filtering not compatible"
1716 if (!(item_flags & l3m))
1717 return rte_flow_error_set(error, EINVAL,
1718 RTE_FLOW_ERROR_TYPE_ITEM, item,
1719 "L3 is mandatory to filter on L4");
1720 if (item_flags & l4m)
1721 return rte_flow_error_set(error, EINVAL,
1722 RTE_FLOW_ERROR_TYPE_ITEM, item,
1723 "multiple L4 layers not supported");
1725 mask = &rte_flow_item_udp_mask;
1726 ret = mlx5_flow_item_acceptable
1727 (item, (const uint8_t *)mask,
1728 (const uint8_t *)&rte_flow_item_udp_mask,
1729 sizeof(struct rte_flow_item_udp), error);
1736 * Validate TCP item.
1739 * Item specification.
1740 * @param[in] item_flags
1741 * Bit-fields that holds the items detected until now.
1742 * @param[in] target_protocol
1743 * The next protocol in the previous item.
1745 * Pointer to error structure.
1748 * 0 on success, a negative errno value otherwise and rte_errno is set.
1751 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
1752 uint64_t item_flags,
1753 uint8_t target_protocol,
1754 const struct rte_flow_item_tcp *flow_mask,
1755 struct rte_flow_error *error)
1757 const struct rte_flow_item_tcp *mask = item->mask;
1758 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1759 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1760 MLX5_FLOW_LAYER_OUTER_L3;
1761 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1762 MLX5_FLOW_LAYER_OUTER_L4;
1765 MLX5_ASSERT(flow_mask);
1766 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
1767 return rte_flow_error_set(error, EINVAL,
1768 RTE_FLOW_ERROR_TYPE_ITEM, item,
1769 "protocol filtering not compatible"
1771 if (!(item_flags & l3m))
1772 return rte_flow_error_set(error, EINVAL,
1773 RTE_FLOW_ERROR_TYPE_ITEM, item,
1774 "L3 is mandatory to filter on L4");
1775 if (item_flags & l4m)
1776 return rte_flow_error_set(error, EINVAL,
1777 RTE_FLOW_ERROR_TYPE_ITEM, item,
1778 "multiple L4 layers not supported");
1780 mask = &rte_flow_item_tcp_mask;
1781 ret = mlx5_flow_item_acceptable
1782 (item, (const uint8_t *)mask,
1783 (const uint8_t *)flow_mask,
1784 sizeof(struct rte_flow_item_tcp), error);
1791 * Validate VXLAN item.
1794 * Item specification.
1795 * @param[in] item_flags
1796 * Bit-fields that holds the items detected until now.
1797 * @param[in] target_protocol
1798 * The next protocol in the previous item.
1800 * Pointer to error structure.
1803 * 0 on success, a negative errno value otherwise and rte_errno is set.
1806 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
1807 uint64_t item_flags,
1808 struct rte_flow_error *error)
1810 const struct rte_flow_item_vxlan *spec = item->spec;
1811 const struct rte_flow_item_vxlan *mask = item->mask;
1816 } id = { .vlan_id = 0, };
1819 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1820 return rte_flow_error_set(error, ENOTSUP,
1821 RTE_FLOW_ERROR_TYPE_ITEM, item,
1822 "multiple tunnel layers not"
1825 * Verify only UDPv4 is present as defined in
1826 * https://tools.ietf.org/html/rfc7348
1828 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1829 return rte_flow_error_set(error, EINVAL,
1830 RTE_FLOW_ERROR_TYPE_ITEM, item,
1831 "no outer UDP layer found");
1833 mask = &rte_flow_item_vxlan_mask;
1834 ret = mlx5_flow_item_acceptable
1835 (item, (const uint8_t *)mask,
1836 (const uint8_t *)&rte_flow_item_vxlan_mask,
1837 sizeof(struct rte_flow_item_vxlan),
1842 memcpy(&id.vni[1], spec->vni, 3);
1843 memcpy(&id.vni[1], mask->vni, 3);
1845 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1846 return rte_flow_error_set(error, ENOTSUP,
1847 RTE_FLOW_ERROR_TYPE_ITEM, item,
1848 "VXLAN tunnel must be fully defined");
1853 * Validate VXLAN_GPE item.
1856 * Item specification.
1857 * @param[in] item_flags
1858 * Bit-fields that holds the items detected until now.
1860 * Pointer to the private data structure.
1861 * @param[in] target_protocol
1862 * The next protocol in the previous item.
1864 * Pointer to error structure.
1867 * 0 on success, a negative errno value otherwise and rte_errno is set.
1870 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
1871 uint64_t item_flags,
1872 struct rte_eth_dev *dev,
1873 struct rte_flow_error *error)
1875 struct mlx5_priv *priv = dev->data->dev_private;
1876 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
1877 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
1882 } id = { .vlan_id = 0, };
1884 if (!priv->config.l3_vxlan_en)
1885 return rte_flow_error_set(error, ENOTSUP,
1886 RTE_FLOW_ERROR_TYPE_ITEM, item,
1887 "L3 VXLAN is not enabled by device"
1888 " parameter and/or not configured in"
1890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1891 return rte_flow_error_set(error, ENOTSUP,
1892 RTE_FLOW_ERROR_TYPE_ITEM, item,
1893 "multiple tunnel layers not"
1896 * Verify only UDPv4 is present as defined in
1897 * https://tools.ietf.org/html/rfc7348
1899 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1900 return rte_flow_error_set(error, EINVAL,
1901 RTE_FLOW_ERROR_TYPE_ITEM, item,
1902 "no outer UDP layer found");
1904 mask = &rte_flow_item_vxlan_gpe_mask;
1905 ret = mlx5_flow_item_acceptable
1906 (item, (const uint8_t *)mask,
1907 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
1908 sizeof(struct rte_flow_item_vxlan_gpe),
1914 return rte_flow_error_set(error, ENOTSUP,
1915 RTE_FLOW_ERROR_TYPE_ITEM,
1917 "VxLAN-GPE protocol"
1919 memcpy(&id.vni[1], spec->vni, 3);
1920 memcpy(&id.vni[1], mask->vni, 3);
1922 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1923 return rte_flow_error_set(error, ENOTSUP,
1924 RTE_FLOW_ERROR_TYPE_ITEM, item,
1925 "VXLAN-GPE tunnel must be fully"
1930 * Validate GRE Key item.
1933 * Item specification.
1934 * @param[in] item_flags
1935 * Bit flags to mark detected items.
1936 * @param[in] gre_item
1937 * Pointer to gre_item
1939 * Pointer to error structure.
1942 * 0 on success, a negative errno value otherwise and rte_errno is set.
1945 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
1946 uint64_t item_flags,
1947 const struct rte_flow_item *gre_item,
1948 struct rte_flow_error *error)
1950 const rte_be32_t *mask = item->mask;
1952 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
1953 const struct rte_flow_item_gre *gre_spec;
1954 const struct rte_flow_item_gre *gre_mask;
1956 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
1957 return rte_flow_error_set(error, ENOTSUP,
1958 RTE_FLOW_ERROR_TYPE_ITEM, item,
1959 "Multiple GRE key not support");
1960 if (!(item_flags & MLX5_FLOW_LAYER_GRE))
1961 return rte_flow_error_set(error, ENOTSUP,
1962 RTE_FLOW_ERROR_TYPE_ITEM, item,
1963 "No preceding GRE header");
1964 if (item_flags & MLX5_FLOW_LAYER_INNER)
1965 return rte_flow_error_set(error, ENOTSUP,
1966 RTE_FLOW_ERROR_TYPE_ITEM, item,
1967 "GRE key following a wrong item");
1968 gre_mask = gre_item->mask;
1970 gre_mask = &rte_flow_item_gre_mask;
1971 gre_spec = gre_item->spec;
1972 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
1973 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
1974 return rte_flow_error_set(error, EINVAL,
1975 RTE_FLOW_ERROR_TYPE_ITEM, item,
1976 "Key bit must be on");
1979 mask = &gre_key_default_mask;
1980 ret = mlx5_flow_item_acceptable
1981 (item, (const uint8_t *)mask,
1982 (const uint8_t *)&gre_key_default_mask,
1983 sizeof(rte_be32_t), error);
1988 * Validate GRE item.
1991 * Item specification.
1992 * @param[in] item_flags
1993 * Bit flags to mark detected items.
1994 * @param[in] target_protocol
1995 * The next protocol in the previous item.
1997 * Pointer to error structure.
2000 * 0 on success, a negative errno value otherwise and rte_errno is set.
2003 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2004 uint64_t item_flags,
2005 uint8_t target_protocol,
2006 struct rte_flow_error *error)
2008 const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2009 const struct rte_flow_item_gre *mask = item->mask;
2011 const struct rte_flow_item_gre nic_mask = {
2012 .c_rsvd0_ver = RTE_BE16(0xB000),
2013 .protocol = RTE_BE16(UINT16_MAX),
2016 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2017 return rte_flow_error_set(error, EINVAL,
2018 RTE_FLOW_ERROR_TYPE_ITEM, item,
2019 "protocol filtering not compatible"
2020 " with this GRE layer");
2021 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2022 return rte_flow_error_set(error, ENOTSUP,
2023 RTE_FLOW_ERROR_TYPE_ITEM, item,
2024 "multiple tunnel layers not"
2026 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2027 return rte_flow_error_set(error, ENOTSUP,
2028 RTE_FLOW_ERROR_TYPE_ITEM, item,
2029 "L3 Layer is missing");
2031 mask = &rte_flow_item_gre_mask;
2032 ret = mlx5_flow_item_acceptable
2033 (item, (const uint8_t *)mask,
2034 (const uint8_t *)&nic_mask,
2035 sizeof(struct rte_flow_item_gre), error);
2038 #ifndef HAVE_MLX5DV_DR
2039 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2040 if (spec && (spec->protocol & mask->protocol))
2041 return rte_flow_error_set(error, ENOTSUP,
2042 RTE_FLOW_ERROR_TYPE_ITEM, item,
2043 "without MPLS support the"
2044 " specification cannot be used for"
2052 * Validate Geneve item.
2055 * Item specification.
2056 * @param[in] itemFlags
2057 * Bit-fields that holds the items detected until now.
2059 * Pointer to the private data structure.
2061 * Pointer to error structure.
2064 * 0 on success, a negative errno value otherwise and rte_errno is set.
2068 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2069 uint64_t item_flags,
2070 struct rte_eth_dev *dev,
2071 struct rte_flow_error *error)
2073 struct mlx5_priv *priv = dev->data->dev_private;
2074 const struct rte_flow_item_geneve *spec = item->spec;
2075 const struct rte_flow_item_geneve *mask = item->mask;
2078 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2079 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2080 const struct rte_flow_item_geneve nic_mask = {
2081 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2082 .vni = "\xff\xff\xff",
2083 .protocol = RTE_BE16(UINT16_MAX),
2086 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2087 return rte_flow_error_set(error, ENOTSUP,
2088 RTE_FLOW_ERROR_TYPE_ITEM, item,
2089 "L3 Geneve is not enabled by device"
2090 " parameter and/or not configured in"
2092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2093 return rte_flow_error_set(error, ENOTSUP,
2094 RTE_FLOW_ERROR_TYPE_ITEM, item,
2095 "multiple tunnel layers not"
2098 * Verify only UDPv4 is present as defined in
2099 * https://tools.ietf.org/html/rfc7348
2101 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2102 return rte_flow_error_set(error, EINVAL,
2103 RTE_FLOW_ERROR_TYPE_ITEM, item,
2104 "no outer UDP layer found");
2106 mask = &rte_flow_item_geneve_mask;
2107 ret = mlx5_flow_item_acceptable
2108 (item, (const uint8_t *)mask,
2109 (const uint8_t *)&nic_mask,
2110 sizeof(struct rte_flow_item_geneve), error);
2114 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2115 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2116 MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2117 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2118 return rte_flow_error_set(error, ENOTSUP,
2119 RTE_FLOW_ERROR_TYPE_ITEM,
2121 "Geneve protocol unsupported"
2122 " fields are being used");
2123 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2124 return rte_flow_error_set
2126 RTE_FLOW_ERROR_TYPE_ITEM,
2128 "Unsupported Geneve options length");
2130 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2131 return rte_flow_error_set
2133 RTE_FLOW_ERROR_TYPE_ITEM, item,
2134 "Geneve tunnel must be fully defined");
2139 * Validate MPLS item.
2142 * Pointer to the rte_eth_dev structure.
2144 * Item specification.
2145 * @param[in] item_flags
2146 * Bit-fields that holds the items detected until now.
2147 * @param[in] prev_layer
2148 * The protocol layer indicated in previous item.
2150 * Pointer to error structure.
2153 * 0 on success, a negative errno value otherwise and rte_errno is set.
2156 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2157 const struct rte_flow_item *item __rte_unused,
2158 uint64_t item_flags __rte_unused,
2159 uint64_t prev_layer __rte_unused,
2160 struct rte_flow_error *error)
2162 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2163 const struct rte_flow_item_mpls *mask = item->mask;
2164 struct mlx5_priv *priv = dev->data->dev_private;
2167 if (!priv->config.mpls_en)
2168 return rte_flow_error_set(error, ENOTSUP,
2169 RTE_FLOW_ERROR_TYPE_ITEM, item,
2170 "MPLS not supported or"
2171 " disabled in firmware"
2173 /* MPLS over IP, UDP, GRE is allowed */
2174 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2175 MLX5_FLOW_LAYER_OUTER_L4_UDP |
2176 MLX5_FLOW_LAYER_GRE)))
2177 return rte_flow_error_set(error, EINVAL,
2178 RTE_FLOW_ERROR_TYPE_ITEM, item,
2179 "protocol filtering not compatible"
2180 " with MPLS layer");
2181 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2182 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2183 !(item_flags & MLX5_FLOW_LAYER_GRE))
2184 return rte_flow_error_set(error, ENOTSUP,
2185 RTE_FLOW_ERROR_TYPE_ITEM, item,
2186 "multiple tunnel layers not"
2189 mask = &rte_flow_item_mpls_mask;
2190 ret = mlx5_flow_item_acceptable
2191 (item, (const uint8_t *)mask,
2192 (const uint8_t *)&rte_flow_item_mpls_mask,
2193 sizeof(struct rte_flow_item_mpls), error);
2198 return rte_flow_error_set(error, ENOTSUP,
2199 RTE_FLOW_ERROR_TYPE_ITEM, item,
2200 "MPLS is not supported by Verbs, please"
2206 * Validate NVGRE item.
2209 * Item specification.
2210 * @param[in] item_flags
2211 * Bit flags to mark detected items.
2212 * @param[in] target_protocol
2213 * The next protocol in the previous item.
2215 * Pointer to error structure.
2218 * 0 on success, a negative errno value otherwise and rte_errno is set.
2221 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2222 uint64_t item_flags,
2223 uint8_t target_protocol,
2224 struct rte_flow_error *error)
2226 const struct rte_flow_item_nvgre *mask = item->mask;
2229 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2230 return rte_flow_error_set(error, EINVAL,
2231 RTE_FLOW_ERROR_TYPE_ITEM, item,
2232 "protocol filtering not compatible"
2233 " with this GRE layer");
2234 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2235 return rte_flow_error_set(error, ENOTSUP,
2236 RTE_FLOW_ERROR_TYPE_ITEM, item,
2237 "multiple tunnel layers not"
2239 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2240 return rte_flow_error_set(error, ENOTSUP,
2241 RTE_FLOW_ERROR_TYPE_ITEM, item,
2242 "L3 Layer is missing");
2244 mask = &rte_flow_item_nvgre_mask;
2245 ret = mlx5_flow_item_acceptable
2246 (item, (const uint8_t *)mask,
2247 (const uint8_t *)&rte_flow_item_nvgre_mask,
2248 sizeof(struct rte_flow_item_nvgre), error);
2255 * Validate eCPRI item.
2258 * Item specification.
2259 * @param[in] item_flags
2260 * Bit-fields that holds the items detected until now.
2261 * @param[in] last_item
2262 * Previous validated item in the pattern items.
2263 * @param[in] ether_type
2264 * Type in the ethernet layer header (including dot1q).
2265 * @param[in] acc_mask
2266 * Acceptable mask, if NULL default internal default mask
2267 * will be used to check whether item fields are supported.
2269 * Pointer to error structure.
2272 * 0 on success, a negative errno value otherwise and rte_errno is set.
2275 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
2276 uint64_t item_flags,
2278 uint16_t ether_type,
2279 const struct rte_flow_item_ecpri *acc_mask,
2280 struct rte_flow_error *error)
2282 const struct rte_flow_item_ecpri *mask = item->mask;
2283 const struct rte_flow_item_ecpri nic_mask = {
2287 RTE_BE32(((const struct rte_ecpri_common_hdr) {
2291 .dummy[0] = 0xFFFFFFFF,
2294 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
2295 MLX5_FLOW_LAYER_OUTER_VLAN);
2296 struct rte_flow_item_ecpri mask_lo;
2298 if ((last_item & outer_l2_vlan) && ether_type &&
2299 ether_type != RTE_ETHER_TYPE_ECPRI)
2300 return rte_flow_error_set(error, EINVAL,
2301 RTE_FLOW_ERROR_TYPE_ITEM, item,
2302 "eCPRI cannot follow L2/VLAN layer "
2303 "which ether type is not 0xAEFE.");
2304 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2305 return rte_flow_error_set(error, EINVAL,
2306 RTE_FLOW_ERROR_TYPE_ITEM, item,
2307 "eCPRI with tunnel is not supported "
2309 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
2310 return rte_flow_error_set(error, ENOTSUP,
2311 RTE_FLOW_ERROR_TYPE_ITEM, item,
2312 "multiple L3 layers not supported");
2313 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
2314 return rte_flow_error_set(error, EINVAL,
2315 RTE_FLOW_ERROR_TYPE_ITEM, item,
2316 "eCPRI cannot follow a TCP layer.");
2317 /* In specification, eCPRI could be over UDP layer. */
2318 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
2319 return rte_flow_error_set(error, EINVAL,
2320 RTE_FLOW_ERROR_TYPE_ITEM, item,
2321 "eCPRI over UDP layer is not yet "
2322 "supported right now.");
2323 /* Mask for type field in common header could be zero. */
2325 mask = &rte_flow_item_ecpri_mask;
2326 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
2327 /* Input mask is in big-endian format. */
2328 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
2329 return rte_flow_error_set(error, EINVAL,
2330 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2331 "partial mask is not supported "
2333 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
2334 return rte_flow_error_set(error, EINVAL,
2335 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2336 "message header mask must be after "
2338 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2339 acc_mask ? (const uint8_t *)acc_mask
2340 : (const uint8_t *)&nic_mask,
2341 sizeof(struct rte_flow_item_ecpri),
2345 /* Allocate unique ID for the split Q/RSS subflows. */
2347 flow_qrss_get_id(struct rte_eth_dev *dev)
2349 struct mlx5_priv *priv = dev->data->dev_private;
2350 uint32_t qrss_id, ret;
2352 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id);
2355 MLX5_ASSERT(qrss_id);
2359 /* Free unique ID for the split Q/RSS subflows. */
2361 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id)
2363 struct mlx5_priv *priv = dev->data->dev_private;
2366 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id);
2370 * Release resource related QUEUE/RSS action split.
2373 * Pointer to Ethernet device.
2375 * Flow to release id's from.
2378 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2379 struct rte_flow *flow)
2381 struct mlx5_priv *priv = dev->data->dev_private;
2382 uint32_t handle_idx;
2383 struct mlx5_flow_handle *dev_handle;
2385 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2386 handle_idx, dev_handle, next)
2387 if (dev_handle->split_flow_id)
2388 flow_qrss_free_id(dev, dev_handle->split_flow_id);
2392 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2393 const struct rte_flow_attr *attr __rte_unused,
2394 const struct rte_flow_item items[] __rte_unused,
2395 const struct rte_flow_action actions[] __rte_unused,
2396 bool external __rte_unused,
2397 int hairpin __rte_unused,
2398 struct rte_flow_error *error)
2400 return rte_flow_error_set(error, ENOTSUP,
2401 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2404 static struct mlx5_flow *
2405 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2406 const struct rte_flow_attr *attr __rte_unused,
2407 const struct rte_flow_item items[] __rte_unused,
2408 const struct rte_flow_action actions[] __rte_unused,
2409 struct rte_flow_error *error)
2411 rte_flow_error_set(error, ENOTSUP,
2412 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2417 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2418 struct mlx5_flow *dev_flow __rte_unused,
2419 const struct rte_flow_attr *attr __rte_unused,
2420 const struct rte_flow_item items[] __rte_unused,
2421 const struct rte_flow_action actions[] __rte_unused,
2422 struct rte_flow_error *error)
2424 return rte_flow_error_set(error, ENOTSUP,
2425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2429 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2430 struct rte_flow *flow __rte_unused,
2431 struct rte_flow_error *error)
2433 return rte_flow_error_set(error, ENOTSUP,
2434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2438 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2439 struct rte_flow *flow __rte_unused)
2444 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2445 struct rte_flow *flow __rte_unused)
2450 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2451 struct rte_flow *flow __rte_unused,
2452 const struct rte_flow_action *actions __rte_unused,
2453 void *data __rte_unused,
2454 struct rte_flow_error *error)
2456 return rte_flow_error_set(error, ENOTSUP,
2457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2460 /* Void driver to protect from null pointer reference. */
2461 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2462 .validate = flow_null_validate,
2463 .prepare = flow_null_prepare,
2464 .translate = flow_null_translate,
2465 .apply = flow_null_apply,
2466 .remove = flow_null_remove,
2467 .destroy = flow_null_destroy,
2468 .query = flow_null_query,
2472 * Select flow driver type according to flow attributes and device
2476 * Pointer to the dev structure.
2478 * Pointer to the flow attributes.
2481 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2483 static enum mlx5_flow_drv_type
2484 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2486 struct mlx5_priv *priv = dev->data->dev_private;
2487 /* The OS can determine first a specific flow type (DV, VERBS) */
2488 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
2490 if (type != MLX5_FLOW_TYPE_MAX)
2492 /* If no OS specific type - continue with DV/VERBS selection */
2493 if (attr->transfer && priv->config.dv_esw_en)
2494 type = MLX5_FLOW_TYPE_DV;
2495 if (!attr->transfer)
2496 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2497 MLX5_FLOW_TYPE_VERBS;
2501 #define flow_get_drv_ops(type) flow_drv_ops[type]
2504 * Flow driver validation API. This abstracts calling driver specific functions.
2505 * The type of flow driver is determined according to flow attributes.
2508 * Pointer to the dev structure.
2510 * Pointer to the flow attributes.
2512 * Pointer to the list of items.
2513 * @param[in] actions
2514 * Pointer to the list of actions.
2515 * @param[in] external
2516 * This flow rule is created by request external to PMD.
2517 * @param[in] hairpin
2518 * Number of hairpin TX actions, 0 means classic flow.
2520 * Pointer to the error structure.
2523 * 0 on success, a negative errno value otherwise and rte_errno is set.
2526 flow_drv_validate(struct rte_eth_dev *dev,
2527 const struct rte_flow_attr *attr,
2528 const struct rte_flow_item items[],
2529 const struct rte_flow_action actions[],
2530 bool external, int hairpin, struct rte_flow_error *error)
2532 const struct mlx5_flow_driver_ops *fops;
2533 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2535 fops = flow_get_drv_ops(type);
2536 return fops->validate(dev, attr, items, actions, external,
2541 * Flow driver preparation API. This abstracts calling driver specific
2542 * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2543 * calculates the size of memory required for device flow, allocates the memory,
2544 * initializes the device flow and returns the pointer.
2547 * This function initializes device flow structure such as dv or verbs in
2548 * struct mlx5_flow. However, it is caller's responsibility to initialize the
2549 * rest. For example, adding returning device flow to flow->dev_flow list and
2550 * setting backward reference to the flow should be done out of this function.
2551 * layers field is not filled either.
2554 * Pointer to the dev structure.
2556 * Pointer to the flow attributes.
2558 * Pointer to the list of items.
2559 * @param[in] actions
2560 * Pointer to the list of actions.
2561 * @param[in] flow_idx
2562 * This memory pool index to the flow.
2564 * Pointer to the error structure.
2567 * Pointer to device flow on success, otherwise NULL and rte_errno is set.
2569 static inline struct mlx5_flow *
2570 flow_drv_prepare(struct rte_eth_dev *dev,
2571 const struct rte_flow *flow,
2572 const struct rte_flow_attr *attr,
2573 const struct rte_flow_item items[],
2574 const struct rte_flow_action actions[],
2576 struct rte_flow_error *error)
2578 const struct mlx5_flow_driver_ops *fops;
2579 enum mlx5_flow_drv_type type = flow->drv_type;
2580 struct mlx5_flow *mlx5_flow = NULL;
2582 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2583 fops = flow_get_drv_ops(type);
2584 mlx5_flow = fops->prepare(dev, attr, items, actions, error);
2586 mlx5_flow->flow_idx = flow_idx;
2591 * Flow driver translation API. This abstracts calling driver specific
2592 * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2593 * translates a generic flow into a driver flow. flow_drv_prepare() must
2597 * dev_flow->layers could be filled as a result of parsing during translation
2598 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
2599 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
2600 * flow->actions could be overwritten even though all the expanded dev_flows
2601 * have the same actions.
2604 * Pointer to the rte dev structure.
2605 * @param[in, out] dev_flow
2606 * Pointer to the mlx5 flow.
2608 * Pointer to the flow attributes.
2610 * Pointer to the list of items.
2611 * @param[in] actions
2612 * Pointer to the list of actions.
2614 * Pointer to the error structure.
2617 * 0 on success, a negative errno value otherwise and rte_errno is set.
2620 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
2621 const struct rte_flow_attr *attr,
2622 const struct rte_flow_item items[],
2623 const struct rte_flow_action actions[],
2624 struct rte_flow_error *error)
2626 const struct mlx5_flow_driver_ops *fops;
2627 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
2629 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2630 fops = flow_get_drv_ops(type);
2631 return fops->translate(dev, dev_flow, attr, items, actions, error);
2635 * Flow driver apply API. This abstracts calling driver specific functions.
2636 * Parent flow (rte_flow) should have driver type (drv_type). It applies
2637 * translated driver flows on to device. flow_drv_translate() must precede.
2640 * Pointer to Ethernet device structure.
2641 * @param[in, out] flow
2642 * Pointer to flow structure.
2644 * Pointer to error structure.
2647 * 0 on success, a negative errno value otherwise and rte_errno is set.
2650 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
2651 struct rte_flow_error *error)
2653 const struct mlx5_flow_driver_ops *fops;
2654 enum mlx5_flow_drv_type type = flow->drv_type;
2656 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2657 fops = flow_get_drv_ops(type);
2658 return fops->apply(dev, flow, error);
2662 * Flow driver remove API. This abstracts calling driver specific functions.
2663 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2664 * on device. All the resources of the flow should be freed by calling
2665 * flow_drv_destroy().
2668 * Pointer to Ethernet device.
2669 * @param[in, out] flow
2670 * Pointer to flow structure.
2673 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
2675 const struct mlx5_flow_driver_ops *fops;
2676 enum mlx5_flow_drv_type type = flow->drv_type;
2678 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2679 fops = flow_get_drv_ops(type);
2680 fops->remove(dev, flow);
2684 * Flow driver destroy API. This abstracts calling driver specific functions.
2685 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2686 * on device and releases resources of the flow.
2689 * Pointer to Ethernet device.
2690 * @param[in, out] flow
2691 * Pointer to flow structure.
2694 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
2696 const struct mlx5_flow_driver_ops *fops;
2697 enum mlx5_flow_drv_type type = flow->drv_type;
2699 flow_mreg_split_qrss_release(dev, flow);
2700 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2701 fops = flow_get_drv_ops(type);
2702 fops->destroy(dev, flow);
2706 * Get RSS action from the action list.
2708 * @param[in] actions
2709 * Pointer to the list of actions.
2712 * Pointer to the RSS action if exist, else return NULL.
2714 static const struct rte_flow_action_rss*
2715 flow_get_rss_action(const struct rte_flow_action actions[])
2717 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2718 switch (actions->type) {
2719 case RTE_FLOW_ACTION_TYPE_RSS:
2720 return (const struct rte_flow_action_rss *)
2730 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
2732 const struct rte_flow_item *item;
2733 unsigned int has_vlan = 0;
2735 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
2736 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
2742 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
2743 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
2744 return rss_level < 2 ? MLX5_EXPANSION_ROOT :
2745 MLX5_EXPANSION_ROOT_OUTER;
2749 * Get layer flags from the prefix flow.
2751 * Some flows may be split to several subflows, the prefix subflow gets the
2752 * match items and the suffix sub flow gets the actions.
2753 * Some actions need the user defined match item flags to get the detail for
2755 * This function helps the suffix flow to get the item layer flags from prefix
2758 * @param[in] dev_flow
2759 * Pointer the created preifx subflow.
2762 * The layers get from prefix subflow.
2764 static inline uint64_t
2765 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
2767 uint64_t layers = 0;
2770 * Layers bits could be localization, but usually the compiler will
2771 * help to do the optimization work for source code.
2772 * If no decap actions, use the layers directly.
2774 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
2775 return dev_flow->handle->layers;
2776 /* Convert L3 layers with decap action. */
2777 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
2778 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2779 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
2780 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2781 /* Convert L4 layers with decap action. */
2782 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
2783 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
2784 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
2785 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
2790 * Get metadata split action information.
2792 * @param[in] actions
2793 * Pointer to the list of actions.
2795 * Pointer to the return pointer.
2796 * @param[out] qrss_type
2797 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
2798 * if no QUEUE/RSS is found.
2799 * @param[out] encap_idx
2800 * Pointer to the index of the encap action if exists, otherwise the last
2804 * Total number of actions.
2807 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
2808 const struct rte_flow_action **qrss,
2811 const struct rte_flow_action_raw_encap *raw_encap;
2813 int raw_decap_idx = -1;
2816 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2817 switch (actions->type) {
2818 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2819 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2820 *encap_idx = actions_n;
2822 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
2823 raw_decap_idx = actions_n;
2825 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2826 raw_encap = actions->conf;
2827 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2828 *encap_idx = raw_decap_idx != -1 ?
2829 raw_decap_idx : actions_n;
2831 case RTE_FLOW_ACTION_TYPE_QUEUE:
2832 case RTE_FLOW_ACTION_TYPE_RSS:
2840 if (*encap_idx == -1)
2841 *encap_idx = actions_n;
2842 /* Count RTE_FLOW_ACTION_TYPE_END. */
2843 return actions_n + 1;
2847 * Check meter action from the action list.
2849 * @param[in] actions
2850 * Pointer to the list of actions.
2852 * Pointer to the meter exist flag.
2855 * Total number of actions.
2858 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
2864 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2865 switch (actions->type) {
2866 case RTE_FLOW_ACTION_TYPE_METER:
2874 /* Count RTE_FLOW_ACTION_TYPE_END. */
2875 return actions_n + 1;
2879 * Check if the flow should be splited due to hairpin.
2880 * The reason for the split is that in current HW we can't
2881 * support encap on Rx, so if a flow have encap we move it
2885 * Pointer to Ethernet device.
2887 * Flow rule attributes.
2888 * @param[in] actions
2889 * Associated actions (list terminated by the END action).
2892 * > 0 the number of actions and the flow should be split,
2893 * 0 when no split required.
2896 flow_check_hairpin_split(struct rte_eth_dev *dev,
2897 const struct rte_flow_attr *attr,
2898 const struct rte_flow_action actions[])
2900 int queue_action = 0;
2903 const struct rte_flow_action_queue *queue;
2904 const struct rte_flow_action_rss *rss;
2905 const struct rte_flow_action_raw_encap *raw_encap;
2909 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2910 switch (actions->type) {
2911 case RTE_FLOW_ACTION_TYPE_QUEUE:
2912 queue = actions->conf;
2915 if (mlx5_rxq_get_type(dev, queue->index) !=
2916 MLX5_RXQ_TYPE_HAIRPIN)
2921 case RTE_FLOW_ACTION_TYPE_RSS:
2922 rss = actions->conf;
2923 if (rss == NULL || rss->queue_num == 0)
2925 if (mlx5_rxq_get_type(dev, rss->queue[0]) !=
2926 MLX5_RXQ_TYPE_HAIRPIN)
2931 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2932 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2936 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2937 raw_encap = actions->conf;
2938 if (raw_encap->size >
2939 (sizeof(struct rte_flow_item_eth) +
2940 sizeof(struct rte_flow_item_ipv4)))
2949 if (encap == 1 && queue_action)
2954 /* Declare flow create/destroy prototype in advance. */
2956 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
2957 const struct rte_flow_attr *attr,
2958 const struct rte_flow_item items[],
2959 const struct rte_flow_action actions[],
2960 bool external, struct rte_flow_error *error);
2963 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
2967 * Add a flow of copying flow metadata registers in RX_CP_TBL.
2969 * As mark_id is unique, if there's already a registered flow for the mark_id,
2970 * return by increasing the reference counter of the resource. Otherwise, create
2971 * the resource (mcp_res) and flow.
2974 * - If ingress port is ANY and reg_c[1] is mark_id,
2975 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
2977 * For default flow (zero mark_id), flow is like,
2978 * - If ingress port is ANY,
2979 * reg_b := reg_c[0] and jump to RX_ACT_TBL.
2982 * Pointer to Ethernet device.
2984 * ID of MARK action, zero means default flow for META.
2986 * Perform verbose error reporting if not NULL.
2989 * Associated resource on success, NULL otherwise and rte_errno is set.
2991 static struct mlx5_flow_mreg_copy_resource *
2992 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
2993 struct rte_flow_error *error)
2995 struct mlx5_priv *priv = dev->data->dev_private;
2996 struct rte_flow_attr attr = {
2997 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3000 struct mlx5_rte_flow_item_tag tag_spec = {
3003 struct rte_flow_item items[] = {
3004 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
3006 struct rte_flow_action_mark ftag = {
3009 struct mlx5_flow_action_copy_mreg cp_mreg = {
3013 struct rte_flow_action_jump jump = {
3014 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
3016 struct rte_flow_action actions[] = {
3017 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
3019 struct mlx5_flow_mreg_copy_resource *mcp_res;
3023 /* Fill the register fileds in the flow. */
3024 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3028 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3032 /* Check if already registered. */
3033 MLX5_ASSERT(priv->mreg_cp_tbl);
3034 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
3036 /* For non-default rule. */
3037 if (mark_id != MLX5_DEFAULT_COPY_ID)
3039 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID ||
3040 mcp_res->refcnt == 1);
3043 /* Provide the full width of FLAG specific value. */
3044 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3045 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3046 /* Build a new flow. */
3047 if (mark_id != MLX5_DEFAULT_COPY_ID) {
3048 items[0] = (struct rte_flow_item){
3049 .type = (enum rte_flow_item_type)
3050 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3053 items[1] = (struct rte_flow_item){
3054 .type = RTE_FLOW_ITEM_TYPE_END,
3056 actions[0] = (struct rte_flow_action){
3057 .type = (enum rte_flow_action_type)
3058 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3061 actions[1] = (struct rte_flow_action){
3062 .type = (enum rte_flow_action_type)
3063 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3066 actions[2] = (struct rte_flow_action){
3067 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3070 actions[3] = (struct rte_flow_action){
3071 .type = RTE_FLOW_ACTION_TYPE_END,
3074 /* Default rule, wildcard match. */
3075 attr.priority = MLX5_FLOW_PRIO_RSVD;
3076 items[0] = (struct rte_flow_item){
3077 .type = RTE_FLOW_ITEM_TYPE_END,
3079 actions[0] = (struct rte_flow_action){
3080 .type = (enum rte_flow_action_type)
3081 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3084 actions[1] = (struct rte_flow_action){
3085 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3088 actions[2] = (struct rte_flow_action){
3089 .type = RTE_FLOW_ACTION_TYPE_END,
3092 /* Build a new entry. */
3093 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3100 * The copy Flows are not included in any list. There
3101 * ones are referenced from other Flows and can not
3102 * be applied, removed, deleted in ardbitrary order
3103 * by list traversing.
3105 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3106 actions, false, error);
3107 if (!mcp_res->rix_flow)
3110 mcp_res->hlist_ent.key = mark_id;
3111 ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
3112 &mcp_res->hlist_ent);
3118 if (mcp_res->rix_flow)
3119 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3120 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3125 * Release flow in RX_CP_TBL.
3128 * Pointer to Ethernet device.
3130 * Parent flow for wich copying is provided.
3133 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3134 struct rte_flow *flow)
3136 struct mlx5_flow_mreg_copy_resource *mcp_res;
3137 struct mlx5_priv *priv = dev->data->dev_private;
3139 if (!flow->rix_mreg_copy)
3141 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3142 flow->rix_mreg_copy);
3143 if (!mcp_res || !priv->mreg_cp_tbl)
3145 if (flow->copy_applied) {
3146 MLX5_ASSERT(mcp_res->appcnt);
3147 flow->copy_applied = 0;
3149 if (!mcp_res->appcnt) {
3150 struct rte_flow *mcp_flow = mlx5_ipool_get
3151 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3155 flow_drv_remove(dev, mcp_flow);
3159 * We do not check availability of metadata registers here,
3160 * because copy resources are not allocated in this case.
3162 if (--mcp_res->refcnt)
3164 MLX5_ASSERT(mcp_res->rix_flow);
3165 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3166 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3167 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3168 flow->rix_mreg_copy = 0;
3172 * Start flow in RX_CP_TBL.
3175 * Pointer to Ethernet device.
3177 * Parent flow for wich copying is provided.
3180 * 0 on success, a negative errno value otherwise and rte_errno is set.
3183 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
3184 struct rte_flow *flow)
3186 struct mlx5_flow_mreg_copy_resource *mcp_res;
3187 struct mlx5_priv *priv = dev->data->dev_private;
3190 if (!flow->rix_mreg_copy || flow->copy_applied)
3192 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3193 flow->rix_mreg_copy);
3196 if (!mcp_res->appcnt) {
3197 struct rte_flow *mcp_flow = mlx5_ipool_get
3198 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3202 ret = flow_drv_apply(dev, mcp_flow, NULL);
3208 flow->copy_applied = 1;
3213 * Stop flow in RX_CP_TBL.
3216 * Pointer to Ethernet device.
3218 * Parent flow for wich copying is provided.
3221 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
3222 struct rte_flow *flow)
3224 struct mlx5_flow_mreg_copy_resource *mcp_res;
3225 struct mlx5_priv *priv = dev->data->dev_private;
3227 if (!flow->rix_mreg_copy || !flow->copy_applied)
3229 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3230 flow->rix_mreg_copy);
3233 MLX5_ASSERT(mcp_res->appcnt);
3235 flow->copy_applied = 0;
3236 if (!mcp_res->appcnt) {
3237 struct rte_flow *mcp_flow = mlx5_ipool_get
3238 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3242 flow_drv_remove(dev, mcp_flow);
3247 * Remove the default copy action from RX_CP_TBL.
3250 * Pointer to Ethernet device.
3253 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3255 struct mlx5_flow_mreg_copy_resource *mcp_res;
3256 struct mlx5_priv *priv = dev->data->dev_private;
3258 /* Check if default flow is registered. */
3259 if (!priv->mreg_cp_tbl)
3261 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
3262 MLX5_DEFAULT_COPY_ID);
3265 MLX5_ASSERT(mcp_res->rix_flow);
3266 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3267 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3268 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3272 * Add the default copy action in in RX_CP_TBL.
3275 * Pointer to Ethernet device.
3277 * Perform verbose error reporting if not NULL.
3280 * 0 for success, negative value otherwise and rte_errno is set.
3283 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3284 struct rte_flow_error *error)
3286 struct mlx5_priv *priv = dev->data->dev_private;
3287 struct mlx5_flow_mreg_copy_resource *mcp_res;
3289 /* Check whether extensive metadata feature is engaged. */
3290 if (!priv->config.dv_flow_en ||
3291 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3292 !mlx5_flow_ext_mreg_supported(dev) ||
3293 !priv->sh->dv_regc0_mask)
3295 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3302 * Add a flow of copying flow metadata registers in RX_CP_TBL.
3304 * All the flow having Q/RSS action should be split by
3305 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3306 * performs the following,
3307 * - CQE->flow_tag := reg_c[1] (MARK)
3308 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3309 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3310 * but there should be a flow per each MARK ID set by MARK action.
3312 * For the aforementioned reason, if there's a MARK action in flow's action
3313 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3314 * the MARK ID to CQE's flow_tag like,
3315 * - If reg_c[1] is mark_id,
3316 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3318 * For SET_META action which stores value in reg_c[0], as the destination is
3319 * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3320 * MARK ID means the default flow. The default flow looks like,
3321 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3324 * Pointer to Ethernet device.
3326 * Pointer to flow structure.
3327 * @param[in] actions
3328 * Pointer to the list of actions.
3330 * Perform verbose error reporting if not NULL.
3333 * 0 on success, negative value otherwise and rte_errno is set.
3336 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3337 struct rte_flow *flow,
3338 const struct rte_flow_action *actions,
3339 struct rte_flow_error *error)
3341 struct mlx5_priv *priv = dev->data->dev_private;
3342 struct mlx5_dev_config *config = &priv->config;
3343 struct mlx5_flow_mreg_copy_resource *mcp_res;
3344 const struct rte_flow_action_mark *mark;
3346 /* Check whether extensive metadata feature is engaged. */
3347 if (!config->dv_flow_en ||
3348 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3349 !mlx5_flow_ext_mreg_supported(dev) ||
3350 !priv->sh->dv_regc0_mask)
3352 /* Find MARK action. */
3353 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3354 switch (actions->type) {
3355 case RTE_FLOW_ACTION_TYPE_FLAG:
3356 mcp_res = flow_mreg_add_copy_action
3357 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3360 flow->rix_mreg_copy = mcp_res->idx;
3361 if (dev->data->dev_started) {
3363 flow->copy_applied = 1;
3366 case RTE_FLOW_ACTION_TYPE_MARK:
3367 mark = (const struct rte_flow_action_mark *)
3370 flow_mreg_add_copy_action(dev, mark->id, error);
3373 flow->rix_mreg_copy = mcp_res->idx;
3374 if (dev->data->dev_started) {
3376 flow->copy_applied = 1;
3386 #define MLX5_MAX_SPLIT_ACTIONS 24
3387 #define MLX5_MAX_SPLIT_ITEMS 24
3390 * Split the hairpin flow.
3391 * Since HW can't support encap on Rx we move the encap to Tx.
3392 * If the count action is after the encap then we also
3393 * move the count action. in this case the count will also measure
3397 * Pointer to Ethernet device.
3398 * @param[in] actions
3399 * Associated actions (list terminated by the END action).
3400 * @param[out] actions_rx
3402 * @param[out] actions_tx
3404 * @param[out] pattern_tx
3405 * The pattern items for the Tx flow.
3406 * @param[out] flow_id
3407 * The flow ID connected to this flow.
3413 flow_hairpin_split(struct rte_eth_dev *dev,
3414 const struct rte_flow_action actions[],
3415 struct rte_flow_action actions_rx[],
3416 struct rte_flow_action actions_tx[],
3417 struct rte_flow_item pattern_tx[],
3420 struct mlx5_priv *priv = dev->data->dev_private;
3421 const struct rte_flow_action_raw_encap *raw_encap;
3422 const struct rte_flow_action_raw_decap *raw_decap;
3423 struct mlx5_rte_flow_action_set_tag *set_tag;
3424 struct rte_flow_action *tag_action;
3425 struct mlx5_rte_flow_item_tag *tag_item;
3426 struct rte_flow_item *item;
3430 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
3431 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3432 switch (actions->type) {
3433 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3434 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3435 rte_memcpy(actions_tx, actions,
3436 sizeof(struct rte_flow_action));
3439 case RTE_FLOW_ACTION_TYPE_COUNT:
3441 rte_memcpy(actions_tx, actions,
3442 sizeof(struct rte_flow_action));
3445 rte_memcpy(actions_rx, actions,
3446 sizeof(struct rte_flow_action));
3450 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3451 raw_encap = actions->conf;
3452 if (raw_encap->size >
3453 (sizeof(struct rte_flow_item_eth) +
3454 sizeof(struct rte_flow_item_ipv4))) {
3455 memcpy(actions_tx, actions,
3456 sizeof(struct rte_flow_action));
3460 rte_memcpy(actions_rx, actions,
3461 sizeof(struct rte_flow_action));
3465 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3466 raw_decap = actions->conf;
3467 if (raw_decap->size <
3468 (sizeof(struct rte_flow_item_eth) +
3469 sizeof(struct rte_flow_item_ipv4))) {
3470 memcpy(actions_tx, actions,
3471 sizeof(struct rte_flow_action));
3474 rte_memcpy(actions_rx, actions,
3475 sizeof(struct rte_flow_action));
3480 rte_memcpy(actions_rx, actions,
3481 sizeof(struct rte_flow_action));
3486 /* Add set meta action and end action for the Rx flow. */
3487 tag_action = actions_rx;
3488 tag_action->type = (enum rte_flow_action_type)
3489 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3491 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
3493 set_tag = (void *)actions_rx;
3494 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
3495 MLX5_ASSERT(set_tag->id > REG_NONE);
3496 set_tag->data = *flow_id;
3497 tag_action->conf = set_tag;
3498 /* Create Tx item list. */
3499 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
3500 addr = (void *)&pattern_tx[2];
3502 item->type = (enum rte_flow_item_type)
3503 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
3504 tag_item = (void *)addr;
3505 tag_item->data = *flow_id;
3506 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
3507 MLX5_ASSERT(set_tag->id > REG_NONE);
3508 item->spec = tag_item;
3509 addr += sizeof(struct mlx5_rte_flow_item_tag);
3510 tag_item = (void *)addr;
3511 tag_item->data = UINT32_MAX;
3512 tag_item->id = UINT16_MAX;
3513 item->mask = tag_item;
3514 addr += sizeof(struct mlx5_rte_flow_item_tag);
3517 item->type = RTE_FLOW_ITEM_TYPE_END;
3522 * The last stage of splitting chain, just creates the subflow
3523 * without any modification.
3526 * Pointer to Ethernet device.
3528 * Parent flow structure pointer.
3529 * @param[in, out] sub_flow
3530 * Pointer to return the created subflow, may be NULL.
3531 * @param[in] prefix_layers
3532 * Prefix subflow layers, may be 0.
3534 * Flow rule attributes.
3536 * Pattern specification (list terminated by the END pattern item).
3537 * @param[in] actions
3538 * Associated actions (list terminated by the END action).
3539 * @param[in] external
3540 * This flow rule is created by request external to PMD.
3541 * @param[in] flow_idx
3542 * This memory pool index to the flow.
3544 * Perform verbose error reporting if not NULL.
3546 * 0 on success, negative value otherwise
3549 flow_create_split_inner(struct rte_eth_dev *dev,
3550 struct rte_flow *flow,
3551 struct mlx5_flow **sub_flow,
3552 uint64_t prefix_layers,
3553 const struct rte_flow_attr *attr,
3554 const struct rte_flow_item items[],
3555 const struct rte_flow_action actions[],
3556 bool external, uint32_t flow_idx,
3557 struct rte_flow_error *error)
3559 struct mlx5_flow *dev_flow;
3561 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
3565 dev_flow->flow = flow;
3566 dev_flow->external = external;
3567 /* Subflow object was created, we must include one in the list. */
3568 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
3569 dev_flow->handle, next);
3571 * If dev_flow is as one of the suffix flow, some actions in suffix
3572 * flow may need some user defined item layer flags.
3575 dev_flow->handle->layers = prefix_layers;
3577 *sub_flow = dev_flow;
3578 return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
3582 * Split the meter flow.
3584 * As meter flow will split to three sub flow, other than meter
3585 * action, the other actions make sense to only meter accepts
3586 * the packet. If it need to be dropped, no other additional
3587 * actions should be take.
3589 * One kind of special action which decapsulates the L3 tunnel
3590 * header will be in the prefix sub flow, as not to take the
3591 * L3 tunnel header into account.
3594 * Pointer to Ethernet device.
3596 * Pattern specification (list terminated by the END pattern item).
3597 * @param[out] sfx_items
3598 * Suffix flow match items (list terminated by the END pattern item).
3599 * @param[in] actions
3600 * Associated actions (list terminated by the END action).
3601 * @param[out] actions_sfx
3602 * Suffix flow actions.
3603 * @param[out] actions_pre
3604 * Prefix flow actions.
3605 * @param[out] pattern_sfx
3606 * The pattern items for the suffix flow.
3607 * @param[out] tag_sfx
3608 * Pointer to suffix flow tag.
3614 flow_meter_split_prep(struct rte_eth_dev *dev,
3615 const struct rte_flow_item items[],
3616 struct rte_flow_item sfx_items[],
3617 const struct rte_flow_action actions[],
3618 struct rte_flow_action actions_sfx[],
3619 struct rte_flow_action actions_pre[])
3621 struct rte_flow_action *tag_action = NULL;
3622 struct rte_flow_item *tag_item;
3623 struct mlx5_rte_flow_action_set_tag *set_tag;
3624 struct rte_flow_error error;
3625 const struct rte_flow_action_raw_encap *raw_encap;
3626 const struct rte_flow_action_raw_decap *raw_decap;
3627 struct mlx5_rte_flow_item_tag *tag_spec;
3628 struct mlx5_rte_flow_item_tag *tag_mask;
3630 bool copy_vlan = false;
3632 /* Prepare the actions for prefix and suffix flow. */
3633 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3634 struct rte_flow_action **action_cur = NULL;
3636 switch (actions->type) {
3637 case RTE_FLOW_ACTION_TYPE_METER:
3638 /* Add the extra tag action first. */
3639 tag_action = actions_pre;
3640 tag_action->type = (enum rte_flow_action_type)
3641 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3643 action_cur = &actions_pre;
3645 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3646 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3647 action_cur = &actions_pre;
3649 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3650 raw_encap = actions->conf;
3651 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
3652 action_cur = &actions_pre;
3654 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3655 raw_decap = actions->conf;
3656 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3657 action_cur = &actions_pre;
3659 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3660 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3667 action_cur = &actions_sfx;
3668 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
3671 /* Add end action to the actions. */
3672 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
3673 actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
3676 set_tag = (void *)actions_pre;
3677 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
3679 * Get the id from the qrss_pool to make qrss share the id with meter.
3681 tag_id = flow_qrss_get_id(dev);
3682 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
3684 tag_action->conf = set_tag;
3685 /* Prepare the suffix subflow items. */
3686 tag_item = sfx_items++;
3687 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3688 int item_type = items->type;
3690 switch (item_type) {
3691 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3692 memcpy(sfx_items, items, sizeof(*sfx_items));
3695 case RTE_FLOW_ITEM_TYPE_VLAN:
3697 memcpy(sfx_items, items, sizeof(*sfx_items));
3699 * Convert to internal match item, it is used
3700 * for vlan push and set vid.
3702 sfx_items->type = (enum rte_flow_item_type)
3703 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
3711 sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
3713 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
3714 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
3715 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
3716 tag_mask = tag_spec + 1;
3717 tag_mask->data = 0xffffff00;
3718 tag_item->type = (enum rte_flow_item_type)
3719 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
3720 tag_item->spec = tag_spec;
3721 tag_item->last = NULL;
3722 tag_item->mask = tag_mask;
3727 * Split action list having QUEUE/RSS for metadata register copy.
3729 * Once Q/RSS action is detected in user's action list, the flow action
3730 * should be split in order to copy metadata registers, which will happen in
3732 * - CQE->flow_tag := reg_c[1] (MARK)
3733 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3734 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
3735 * This is because the last action of each flow must be a terminal action
3736 * (QUEUE, RSS or DROP).
3738 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
3739 * stored and kept in the mlx5_flow structure per each sub_flow.
3741 * The Q/RSS action is replaced with,
3742 * - SET_TAG, setting the allocated flow ID to reg_c[2].
3743 * And the following JUMP action is added at the end,
3744 * - JUMP, to RX_CP_TBL.
3746 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
3747 * flow_create_split_metadata() routine. The flow will look like,
3748 * - If flow ID matches (reg_c[2]), perform Q/RSS.
3751 * Pointer to Ethernet device.
3752 * @param[out] split_actions
3753 * Pointer to store split actions to jump to CP_TBL.
3754 * @param[in] actions
3755 * Pointer to the list of original flow actions.
3757 * Pointer to the Q/RSS action.
3758 * @param[in] actions_n
3759 * Number of original actions.
3761 * Perform verbose error reporting if not NULL.
3764 * non-zero unique flow_id on success, otherwise 0 and
3765 * error/rte_error are set.
3768 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
3769 struct rte_flow_action *split_actions,
3770 const struct rte_flow_action *actions,
3771 const struct rte_flow_action *qrss,
3772 int actions_n, struct rte_flow_error *error)
3774 struct mlx5_rte_flow_action_set_tag *set_tag;
3775 struct rte_flow_action_jump *jump;
3776 const int qrss_idx = qrss - actions;
3777 uint32_t flow_id = 0;
3781 * Given actions will be split
3782 * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
3783 * - Add jump to mreg CP_TBL.
3784 * As a result, there will be one more action.
3787 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
3788 set_tag = (void *)(split_actions + actions_n);
3790 * If tag action is not set to void(it means we are not the meter
3791 * suffix flow), add the tag action. Since meter suffix flow already
3792 * has the tag added.
3794 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
3796 * Allocate the new subflow ID. This one is unique within
3797 * device and not shared with representors. Otherwise,
3798 * we would have to resolve multi-thread access synch
3799 * issue. Each flow on the shared device is appended
3800 * with source vport identifier, so the resulting
3801 * flows will be unique in the shared (by master and
3802 * representors) domain even if they have coinciding
3805 flow_id = flow_qrss_get_id(dev);
3807 return rte_flow_error_set(error, ENOMEM,
3808 RTE_FLOW_ERROR_TYPE_ACTION,
3809 NULL, "can't allocate id "
3810 "for split Q/RSS subflow");
3811 /* Internal SET_TAG action to set flow ID. */
3812 *set_tag = (struct mlx5_rte_flow_action_set_tag){
3815 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
3819 /* Construct new actions array. */
3820 /* Replace QUEUE/RSS action. */
3821 split_actions[qrss_idx] = (struct rte_flow_action){
3822 .type = (enum rte_flow_action_type)
3823 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
3827 /* JUMP action to jump to mreg copy table (CP_TBL). */
3828 jump = (void *)(set_tag + 1);
3829 *jump = (struct rte_flow_action_jump){
3830 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3832 split_actions[actions_n - 2] = (struct rte_flow_action){
3833 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3836 split_actions[actions_n - 1] = (struct rte_flow_action){
3837 .type = RTE_FLOW_ACTION_TYPE_END,
3843 * Extend the given action list for Tx metadata copy.
3845 * Copy the given action list to the ext_actions and add flow metadata register
3846 * copy action in order to copy reg_a set by WQE to reg_c[0].
3848 * @param[out] ext_actions
3849 * Pointer to the extended action list.
3850 * @param[in] actions
3851 * Pointer to the list of actions.
3852 * @param[in] actions_n
3853 * Number of actions in the list.
3855 * Perform verbose error reporting if not NULL.
3856 * @param[in] encap_idx
3857 * The encap action inndex.
3860 * 0 on success, negative value otherwise
3863 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
3864 struct rte_flow_action *ext_actions,
3865 const struct rte_flow_action *actions,
3866 int actions_n, struct rte_flow_error *error,
3869 struct mlx5_flow_action_copy_mreg *cp_mreg =
3870 (struct mlx5_flow_action_copy_mreg *)
3871 (ext_actions + actions_n + 1);
3874 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3878 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
3883 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
3884 if (encap_idx == actions_n - 1) {
3885 ext_actions[actions_n - 1] = (struct rte_flow_action){
3886 .type = (enum rte_flow_action_type)
3887 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3890 ext_actions[actions_n] = (struct rte_flow_action){
3891 .type = RTE_FLOW_ACTION_TYPE_END,
3894 ext_actions[encap_idx] = (struct rte_flow_action){
3895 .type = (enum rte_flow_action_type)
3896 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3899 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
3900 sizeof(*ext_actions) * (actions_n - encap_idx));
3906 * The splitting for metadata feature.
3908 * - Q/RSS action on NIC Rx should be split in order to pass by
3909 * the mreg copy table (RX_CP_TBL) and then it jumps to the
3910 * action table (RX_ACT_TBL) which has the split Q/RSS action.
3912 * - All the actions on NIC Tx should have a mreg copy action to
3913 * copy reg_a from WQE to reg_c[0].
3916 * Pointer to Ethernet device.
3918 * Parent flow structure pointer.
3919 * @param[in] prefix_layers
3920 * Prefix flow layer flags.
3922 * Flow rule attributes.
3924 * Pattern specification (list terminated by the END pattern item).
3925 * @param[in] actions
3926 * Associated actions (list terminated by the END action).
3927 * @param[in] external
3928 * This flow rule is created by request external to PMD.
3929 * @param[in] flow_idx
3930 * This memory pool index to the flow.
3932 * Perform verbose error reporting if not NULL.
3934 * 0 on success, negative value otherwise
3937 flow_create_split_metadata(struct rte_eth_dev *dev,
3938 struct rte_flow *flow,
3939 uint64_t prefix_layers,
3940 const struct rte_flow_attr *attr,
3941 const struct rte_flow_item items[],
3942 const struct rte_flow_action actions[],
3943 bool external, uint32_t flow_idx,
3944 struct rte_flow_error *error)
3946 struct mlx5_priv *priv = dev->data->dev_private;
3947 struct mlx5_dev_config *config = &priv->config;
3948 const struct rte_flow_action *qrss = NULL;
3949 struct rte_flow_action *ext_actions = NULL;
3950 struct mlx5_flow *dev_flow = NULL;
3951 uint32_t qrss_id = 0;
3958 /* Check whether extensive metadata feature is engaged. */
3959 if (!config->dv_flow_en ||
3960 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3961 !mlx5_flow_ext_mreg_supported(dev))
3962 return flow_create_split_inner(dev, flow, NULL, prefix_layers,
3963 attr, items, actions, external,
3965 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
3968 /* Exclude hairpin flows from splitting. */
3969 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
3970 const struct rte_flow_action_queue *queue;
3973 if (mlx5_rxq_get_type(dev, queue->index) ==
3974 MLX5_RXQ_TYPE_HAIRPIN)
3976 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
3977 const struct rte_flow_action_rss *rss;
3980 if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
3981 MLX5_RXQ_TYPE_HAIRPIN)
3986 /* Check if it is in meter suffix table. */
3987 mtr_sfx = attr->group == (attr->transfer ?
3988 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
3989 MLX5_FLOW_TABLE_LEVEL_SUFFIX);
3991 * Q/RSS action on NIC Rx should be split in order to pass by
3992 * the mreg copy table (RX_CP_TBL) and then it jumps to the
3993 * action table (RX_ACT_TBL) which has the split Q/RSS action.
3995 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
3996 sizeof(struct rte_flow_action_set_tag) +
3997 sizeof(struct rte_flow_action_jump);
3998 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4001 return rte_flow_error_set(error, ENOMEM,
4002 RTE_FLOW_ERROR_TYPE_ACTION,
4003 NULL, "no memory to split "
4006 * If we are the suffix flow of meter, tag already exist.
4007 * Set the tag action to void.
4010 ext_actions[qrss - actions].type =
4011 RTE_FLOW_ACTION_TYPE_VOID;
4013 ext_actions[qrss - actions].type =
4014 (enum rte_flow_action_type)
4015 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4017 * Create the new actions list with removed Q/RSS action
4018 * and appended set tag and jump to register copy table
4019 * (RX_CP_TBL). We should preallocate unique tag ID here
4020 * in advance, because it is needed for set tag action.
4022 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
4023 qrss, actions_n, error);
4024 if (!mtr_sfx && !qrss_id) {
4028 } else if (attr->egress && !attr->transfer) {
4030 * All the actions on NIC Tx should have a metadata register
4031 * copy action to copy reg_a from WQE to reg_c[meta]
4033 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
4034 sizeof(struct mlx5_flow_action_copy_mreg);
4035 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
4038 return rte_flow_error_set(error, ENOMEM,
4039 RTE_FLOW_ERROR_TYPE_ACTION,
4040 NULL, "no memory to split "
4042 /* Create the action list appended with copy register. */
4043 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4044 actions_n, error, encap_idx);
4048 /* Add the unmodified original or prefix subflow. */
4049 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr,
4050 items, ext_actions ? ext_actions :
4051 actions, external, flow_idx, error);
4054 MLX5_ASSERT(dev_flow);
4056 const struct rte_flow_attr q_attr = {
4057 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4060 /* Internal PMD action to set register. */
4061 struct mlx5_rte_flow_item_tag q_tag_spec = {
4065 struct rte_flow_item q_items[] = {
4067 .type = (enum rte_flow_item_type)
4068 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4069 .spec = &q_tag_spec,
4074 .type = RTE_FLOW_ITEM_TYPE_END,
4077 struct rte_flow_action q_actions[] = {
4083 .type = RTE_FLOW_ACTION_TYPE_END,
4086 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4089 * Configure the tag item only if there is no meter subflow.
4090 * Since tag is already marked in the meter suffix subflow
4091 * we can just use the meter suffix items as is.
4094 /* Not meter subflow. */
4095 MLX5_ASSERT(!mtr_sfx);
4097 * Put unique id in prefix flow due to it is destroyed
4098 * after suffix flow and id will be freed after there
4099 * is no actual flows with this id and identifier
4100 * reallocation becomes possible (for example, for
4101 * other flows in other threads).
4103 dev_flow->handle->split_flow_id = qrss_id;
4104 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4108 q_tag_spec.id = ret;
4111 /* Add suffix subflow to execute Q/RSS. */
4112 ret = flow_create_split_inner(dev, flow, &dev_flow, layers,
4113 &q_attr, mtr_sfx ? items :
4115 external, flow_idx, error);
4118 /* qrss ID should be freed if failed. */
4120 MLX5_ASSERT(dev_flow);
4125 * We do not destroy the partially created sub_flows in case of error.
4126 * These ones are included into parent flow list and will be destroyed
4127 * by flow_drv_destroy.
4129 flow_qrss_free_id(dev, qrss_id);
4130 mlx5_free(ext_actions);
4135 * The splitting for meter feature.
4137 * - The meter flow will be split to two flows as prefix and
4138 * suffix flow. The packets make sense only it pass the prefix
4141 * - Reg_C_5 is used for the packet to match betweend prefix and
4145 * Pointer to Ethernet device.
4147 * Parent flow structure pointer.
4149 * Flow rule attributes.
4151 * Pattern specification (list terminated by the END pattern item).
4152 * @param[in] actions
4153 * Associated actions (list terminated by the END action).
4154 * @param[in] external
4155 * This flow rule is created by request external to PMD.
4156 * @param[in] flow_idx
4157 * This memory pool index to the flow.
4159 * Perform verbose error reporting if not NULL.
4161 * 0 on success, negative value otherwise
4164 flow_create_split_meter(struct rte_eth_dev *dev,
4165 struct rte_flow *flow,
4166 const struct rte_flow_attr *attr,
4167 const struct rte_flow_item items[],
4168 const struct rte_flow_action actions[],
4169 bool external, uint32_t flow_idx,
4170 struct rte_flow_error *error)
4172 struct mlx5_priv *priv = dev->data->dev_private;
4173 struct rte_flow_action *sfx_actions = NULL;
4174 struct rte_flow_action *pre_actions = NULL;
4175 struct rte_flow_item *sfx_items = NULL;
4176 struct mlx5_flow *dev_flow = NULL;
4177 struct rte_flow_attr sfx_attr = *attr;
4179 uint32_t mtr_tag_id = 0;
4186 actions_n = flow_check_meter_action(actions, &mtr);
4188 /* The five prefix actions: meter, decap, encap, tag, end. */
4189 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4190 sizeof(struct mlx5_rte_flow_action_set_tag);
4191 /* tag, vlan, port id, end. */
4192 #define METER_SUFFIX_ITEM 4
4193 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4194 sizeof(struct mlx5_rte_flow_item_tag) * 2;
4195 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
4198 return rte_flow_error_set(error, ENOMEM,
4199 RTE_FLOW_ERROR_TYPE_ACTION,
4200 NULL, "no memory to split "
4202 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4204 pre_actions = sfx_actions + actions_n;
4205 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4206 actions, sfx_actions,
4212 /* Add the prefix subflow. */
4213 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr,
4214 items, pre_actions, external,
4220 dev_flow->handle->split_flow_id = mtr_tag_id;
4221 /* Setting the sfx group atrr. */
4222 sfx_attr.group = sfx_attr.transfer ?
4223 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4224 MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4226 /* Add the prefix subflow. */
4227 ret = flow_create_split_metadata(dev, flow, dev_flow ?
4228 flow_get_prefix_layer_flags(dev_flow) :
4230 sfx_items ? sfx_items : items,
4231 sfx_actions ? sfx_actions : actions,
4232 external, flow_idx, error);
4235 mlx5_free(sfx_actions);
4240 * Split the flow to subflow set. The splitters might be linked
4241 * in the chain, like this:
4242 * flow_create_split_outer() calls:
4243 * flow_create_split_meter() calls:
4244 * flow_create_split_metadata(meter_subflow_0) calls:
4245 * flow_create_split_inner(metadata_subflow_0)
4246 * flow_create_split_inner(metadata_subflow_1)
4247 * flow_create_split_inner(metadata_subflow_2)
4248 * flow_create_split_metadata(meter_subflow_1) calls:
4249 * flow_create_split_inner(metadata_subflow_0)
4250 * flow_create_split_inner(metadata_subflow_1)
4251 * flow_create_split_inner(metadata_subflow_2)
4253 * This provide flexible way to add new levels of flow splitting.
4254 * The all of successfully created subflows are included to the
4255 * parent flow dev_flow list.
4258 * Pointer to Ethernet device.
4260 * Parent flow structure pointer.
4262 * Flow rule attributes.
4264 * Pattern specification (list terminated by the END pattern item).
4265 * @param[in] actions
4266 * Associated actions (list terminated by the END action).
4267 * @param[in] external
4268 * This flow rule is created by request external to PMD.
4269 * @param[in] flow_idx
4270 * This memory pool index to the flow.
4272 * Perform verbose error reporting if not NULL.
4274 * 0 on success, negative value otherwise
4277 flow_create_split_outer(struct rte_eth_dev *dev,
4278 struct rte_flow *flow,
4279 const struct rte_flow_attr *attr,
4280 const struct rte_flow_item items[],
4281 const struct rte_flow_action actions[],
4282 bool external, uint32_t flow_idx,
4283 struct rte_flow_error *error)
4287 ret = flow_create_split_meter(dev, flow, attr, items,
4288 actions, external, flow_idx, error);
4289 MLX5_ASSERT(ret <= 0);
4294 * Create a flow and add it to @p list.
4297 * Pointer to Ethernet device.
4299 * Pointer to a TAILQ flow list. If this parameter NULL,
4300 * no list insertion occurred, flow is just created,
4301 * this is caller's responsibility to track the
4304 * Flow rule attributes.
4306 * Pattern specification (list terminated by the END pattern item).
4307 * @param[in] actions
4308 * Associated actions (list terminated by the END action).
4309 * @param[in] external
4310 * This flow rule is created by request external to PMD.
4312 * Perform verbose error reporting if not NULL.
4315 * A flow index on success, 0 otherwise and rte_errno is set.
4318 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
4319 const struct rte_flow_attr *attr,
4320 const struct rte_flow_item items[],
4321 const struct rte_flow_action actions[],
4322 bool external, struct rte_flow_error *error)
4324 struct mlx5_priv *priv = dev->data->dev_private;
4325 struct rte_flow *flow = NULL;
4326 struct mlx5_flow *dev_flow;
4327 const struct rte_flow_action_rss *rss;
4329 struct rte_flow_expand_rss buf;
4330 uint8_t buffer[2048];
4333 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4334 uint8_t buffer[2048];
4337 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4338 uint8_t buffer[2048];
4339 } actions_hairpin_tx;
4341 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
4342 uint8_t buffer[2048];
4344 struct rte_flow_expand_rss *buf = &expand_buffer.buf;
4345 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
4346 priv->rss_desc)[!!priv->flow_idx];
4347 const struct rte_flow_action *p_actions_rx = actions;
4351 uint32_t hairpin_id = 0;
4352 struct rte_flow_attr attr_tx = { .priority = 0 };
4355 hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4356 ret = flow_drv_validate(dev, attr, items, p_actions_rx,
4357 external, hairpin_flow, error);
4360 if (hairpin_flow > 0) {
4361 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
4365 flow_hairpin_split(dev, actions, actions_rx.actions,
4366 actions_hairpin_tx.actions, items_tx.items,
4368 p_actions_rx = actions_rx.actions;
4370 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
4373 goto error_before_flow;
4375 flow->drv_type = flow_get_drv_type(dev, attr);
4376 if (hairpin_id != 0)
4377 flow->hairpin_flow_id = hairpin_id;
4378 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
4379 flow->drv_type < MLX5_FLOW_TYPE_MAX);
4380 memset(rss_desc, 0, sizeof(*rss_desc));
4381 rss = flow_get_rss_action(p_actions_rx);
4384 * The following information is required by
4385 * mlx5_flow_hashfields_adjust() in advance.
4387 rss_desc->level = rss->level;
4388 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
4389 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
4391 flow->dev_handles = 0;
4392 if (rss && rss->types) {
4393 unsigned int graph_root;
4395 graph_root = find_graph_root(items, rss->level);
4396 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
4398 mlx5_support_expansion,
4400 MLX5_ASSERT(ret > 0 &&
4401 (unsigned int)ret < sizeof(expand_buffer.buffer));
4404 buf->entry[0].pattern = (void *)(uintptr_t)items;
4407 * Record the start index when there is a nested call. All sub-flows
4408 * need to be translated before another calling.
4409 * No need to use ping-pong buffer to save memory here.
4411 if (priv->flow_idx) {
4412 MLX5_ASSERT(!priv->flow_nested_idx);
4413 priv->flow_nested_idx = priv->flow_idx;
4415 for (i = 0; i < buf->entries; ++i) {
4417 * The splitter may create multiple dev_flows,
4418 * depending on configuration. In the simplest
4419 * case it just creates unmodified original flow.
4421 ret = flow_create_split_outer(dev, flow, attr,
4422 buf->entry[i].pattern,
4423 p_actions_rx, external, idx,
4428 /* Create the tx flow. */
4430 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
4431 attr_tx.ingress = 0;
4433 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
4434 actions_hairpin_tx.actions,
4438 dev_flow->flow = flow;
4439 dev_flow->external = 0;
4440 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4441 dev_flow->handle, next);
4442 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
4444 actions_hairpin_tx.actions, error);
4449 * Update the metadata register copy table. If extensive
4450 * metadata feature is enabled and registers are supported
4451 * we might create the extra rte_flow for each unique
4452 * MARK/FLAG action ID.
4454 * The table is updated for ingress Flows only, because
4455 * the egress Flows belong to the different device and
4456 * copy table should be updated in peer NIC Rx domain.
4458 if (attr->ingress &&
4459 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
4460 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
4465 * If the flow is external (from application) OR device is started, then
4466 * the flow will be applied immediately.
4468 if (external || dev->data->dev_started) {
4469 ret = flow_drv_apply(dev, flow, error);
4474 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
4476 flow_rxq_flags_set(dev, flow);
4477 /* Nested flow creation index recovery. */
4478 priv->flow_idx = priv->flow_nested_idx;
4479 if (priv->flow_nested_idx)
4480 priv->flow_nested_idx = 0;
4484 ret = rte_errno; /* Save rte_errno before cleanup. */
4485 flow_mreg_del_copy_action(dev, flow);
4486 flow_drv_destroy(dev, flow);
4487 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
4488 rte_errno = ret; /* Restore rte_errno. */
4492 mlx5_flow_id_release(priv->sh->flow_id_pool,
4495 priv->flow_idx = priv->flow_nested_idx;
4496 if (priv->flow_nested_idx)
4497 priv->flow_nested_idx = 0;
4502 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
4503 * incoming packets to table 1.
4505 * Other flow rules, requested for group n, will be created in
4506 * e-switch table n+1.
4507 * Jump action to e-switch group n will be created to group n+1.
4509 * Used when working in switchdev mode, to utilise advantages of table 1
4513 * Pointer to Ethernet device.
4516 * Pointer to flow on success, NULL otherwise and rte_errno is set.
4519 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
4521 const struct rte_flow_attr attr = {
4528 const struct rte_flow_item pattern = {
4529 .type = RTE_FLOW_ITEM_TYPE_END,
4531 struct rte_flow_action_jump jump = {
4534 const struct rte_flow_action actions[] = {
4536 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4540 .type = RTE_FLOW_ACTION_TYPE_END,
4543 struct mlx5_priv *priv = dev->data->dev_private;
4544 struct rte_flow_error error;
4546 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
4548 actions, false, &error);
4552 * Validate a flow supported by the NIC.
4554 * @see rte_flow_validate()
4558 mlx5_flow_validate(struct rte_eth_dev *dev,
4559 const struct rte_flow_attr *attr,
4560 const struct rte_flow_item items[],
4561 const struct rte_flow_action actions[],
4562 struct rte_flow_error *error)
4566 hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4567 return flow_drv_validate(dev, attr, items, actions,
4568 true, hairpin_flow, error);
4574 * @see rte_flow_create()
4578 mlx5_flow_create(struct rte_eth_dev *dev,
4579 const struct rte_flow_attr *attr,
4580 const struct rte_flow_item items[],
4581 const struct rte_flow_action actions[],
4582 struct rte_flow_error *error)
4584 struct mlx5_priv *priv = dev->data->dev_private;
4587 * If the device is not started yet, it is not allowed to created a
4588 * flow from application. PMD default flows and traffic control flows
4591 if (unlikely(!dev->data->dev_started)) {
4592 DRV_LOG(DEBUG, "port %u is not started when "
4593 "inserting a flow", dev->data->port_id);
4594 rte_flow_error_set(error, ENODEV,
4595 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4597 "port not started");
4600 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
4601 attr, items, actions, true, error);
4605 * Destroy a flow in a list.
4608 * Pointer to Ethernet device.
4610 * Pointer to the Indexed flow list. If this parameter NULL,
4611 * there is no flow removal from the list. Be noted that as
4612 * flow is add to the indexed list, memory of the indexed
4613 * list points to maybe changed as flow destroyed.
4614 * @param[in] flow_idx
4615 * Index of flow to destroy.
4618 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
4621 struct mlx5_priv *priv = dev->data->dev_private;
4622 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
4623 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
4624 [MLX5_IPOOL_RTE_FLOW], flow_idx);
4629 * Update RX queue flags only if port is started, otherwise it is
4632 if (dev->data->dev_started)
4633 flow_rxq_flags_trim(dev, flow);
4634 if (flow->hairpin_flow_id)
4635 mlx5_flow_id_release(priv->sh->flow_id_pool,
4636 flow->hairpin_flow_id);
4637 flow_drv_destroy(dev, flow);
4639 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
4640 flow_idx, flow, next);
4641 flow_mreg_del_copy_action(dev, flow);
4643 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
4644 if (priv_fdir_flow->rix_flow == flow_idx)
4647 if (priv_fdir_flow) {
4648 LIST_REMOVE(priv_fdir_flow, next);
4649 mlx5_free(priv_fdir_flow->fdir);
4650 mlx5_free(priv_fdir_flow);
4653 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
4657 * Destroy all flows.
4660 * Pointer to Ethernet device.
4662 * Pointer to the Indexed flow list.
4664 * If flushing is called avtively.
4667 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
4669 uint32_t num_flushed = 0;
4672 flow_list_destroy(dev, list, *list);
4676 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
4677 dev->data->port_id, num_flushed);
4685 * Pointer to Ethernet device.
4687 * Pointer to the Indexed flow list.
4690 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list)
4692 struct mlx5_priv *priv = dev->data->dev_private;
4693 struct rte_flow *flow = NULL;
4696 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
4698 flow_drv_remove(dev, flow);
4699 flow_mreg_stop_copy_action(dev, flow);
4701 flow_mreg_del_default_copy_action(dev);
4702 flow_rxq_flags_clear(dev);
4709 * Pointer to Ethernet device.
4711 * Pointer to the Indexed flow list.
4714 * 0 on success, a negative errno value otherwise and rte_errno is set.
4717 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list)
4719 struct mlx5_priv *priv = dev->data->dev_private;
4720 struct rte_flow *flow = NULL;
4721 struct rte_flow_error error;
4725 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
4726 ret = flow_mreg_add_default_copy_action(dev, &error);
4729 /* Apply Flows created by application. */
4730 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
4732 ret = flow_mreg_start_copy_action(dev, flow);
4735 ret = flow_drv_apply(dev, flow, &error);
4738 flow_rxq_flags_set(dev, flow);
4742 ret = rte_errno; /* Save rte_errno before cleanup. */
4743 mlx5_flow_stop(dev, list);
4744 rte_errno = ret; /* Restore rte_errno. */
4749 * Stop all default actions for flows.
4752 * Pointer to Ethernet device.
4755 mlx5_flow_stop_default(struct rte_eth_dev *dev)
4757 flow_mreg_del_default_copy_action(dev);
4758 flow_rxq_flags_clear(dev);
4762 * Start all default actions for flows.
4765 * Pointer to Ethernet device.
4767 * 0 on success, a negative errno value otherwise and rte_errno is set.
4770 mlx5_flow_start_default(struct rte_eth_dev *dev)
4772 struct rte_flow_error error;
4774 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
4775 return flow_mreg_add_default_copy_action(dev, &error);
4779 * Allocate intermediate resources for flow creation.
4782 * Pointer to Ethernet device.
4785 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev)
4787 struct mlx5_priv *priv = dev->data->dev_private;
4789 if (!priv->inter_flows) {
4790 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO,
4791 MLX5_NUM_MAX_DEV_FLOWS *
4792 sizeof(struct mlx5_flow) +
4793 (sizeof(struct mlx5_flow_rss_desc) +
4794 sizeof(uint16_t) * UINT16_MAX) * 2, 0,
4796 if (!priv->inter_flows) {
4797 DRV_LOG(ERR, "can't allocate intermediate memory.");
4801 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows)
4802 [MLX5_NUM_MAX_DEV_FLOWS];
4803 /* Reset the index. */
4805 priv->flow_nested_idx = 0;
4809 * Free intermediate resources for flows.
4812 * Pointer to Ethernet device.
4815 mlx5_flow_free_intermediate(struct rte_eth_dev *dev)
4817 struct mlx5_priv *priv = dev->data->dev_private;
4819 mlx5_free(priv->inter_flows);
4820 priv->inter_flows = NULL;
4824 * Verify the flow list is empty
4827 * Pointer to Ethernet device.
4829 * @return the number of flows not released.
4832 mlx5_flow_verify(struct rte_eth_dev *dev)
4834 struct mlx5_priv *priv = dev->data->dev_private;
4835 struct rte_flow *flow;
4839 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
4841 DRV_LOG(DEBUG, "port %u flow %p still referenced",
4842 dev->data->port_id, (void *)flow);
4849 * Enable default hairpin egress flow.
4852 * Pointer to Ethernet device.
4857 * 0 on success, a negative errno value otherwise and rte_errno is set.
4860 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
4863 struct mlx5_priv *priv = dev->data->dev_private;
4864 const struct rte_flow_attr attr = {
4868 struct mlx5_rte_flow_item_tx_queue queue_spec = {
4871 struct mlx5_rte_flow_item_tx_queue queue_mask = {
4872 .queue = UINT32_MAX,
4874 struct rte_flow_item items[] = {
4876 .type = (enum rte_flow_item_type)
4877 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
4878 .spec = &queue_spec,
4880 .mask = &queue_mask,
4883 .type = RTE_FLOW_ITEM_TYPE_END,
4886 struct rte_flow_action_jump jump = {
4887 .group = MLX5_HAIRPIN_TX_TABLE,
4889 struct rte_flow_action actions[2];
4891 struct rte_flow_error error;
4893 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
4894 actions[0].conf = &jump;
4895 actions[1].type = RTE_FLOW_ACTION_TYPE_END;
4896 flow_idx = flow_list_create(dev, &priv->ctrl_flows,
4897 &attr, items, actions, false, &error);
4900 "Failed to create ctrl flow: rte_errno(%d),"
4901 " type(%d), message(%s)",
4902 rte_errno, error.type,
4903 error.message ? error.message : " (no stated reason)");
4910 * Enable a control flow configured from the control plane.
4913 * Pointer to Ethernet device.
4915 * An Ethernet flow spec to apply.
4917 * An Ethernet flow mask to apply.
4919 * A VLAN flow spec to apply.
4921 * A VLAN flow mask to apply.
4924 * 0 on success, a negative errno value otherwise and rte_errno is set.
4927 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
4928 struct rte_flow_item_eth *eth_spec,
4929 struct rte_flow_item_eth *eth_mask,
4930 struct rte_flow_item_vlan *vlan_spec,
4931 struct rte_flow_item_vlan *vlan_mask)
4933 struct mlx5_priv *priv = dev->data->dev_private;
4934 const struct rte_flow_attr attr = {
4936 .priority = MLX5_FLOW_PRIO_RSVD,
4938 struct rte_flow_item items[] = {
4940 .type = RTE_FLOW_ITEM_TYPE_ETH,
4946 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
4947 RTE_FLOW_ITEM_TYPE_END,
4953 .type = RTE_FLOW_ITEM_TYPE_END,
4956 uint16_t queue[priv->reta_idx_n];
4957 struct rte_flow_action_rss action_rss = {
4958 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
4960 .types = priv->rss_conf.rss_hf,
4961 .key_len = priv->rss_conf.rss_key_len,
4962 .queue_num = priv->reta_idx_n,
4963 .key = priv->rss_conf.rss_key,
4966 struct rte_flow_action actions[] = {
4968 .type = RTE_FLOW_ACTION_TYPE_RSS,
4969 .conf = &action_rss,
4972 .type = RTE_FLOW_ACTION_TYPE_END,
4976 struct rte_flow_error error;
4979 if (!priv->reta_idx_n || !priv->rxqs_n) {
4982 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
4983 action_rss.types = 0;
4984 for (i = 0; i != priv->reta_idx_n; ++i)
4985 queue[i] = (*priv->reta_idx)[i];
4986 flow_idx = flow_list_create(dev, &priv->ctrl_flows,
4987 &attr, items, actions, false, &error);
4994 * Enable a flow control configured from the control plane.
4997 * Pointer to Ethernet device.
4999 * An Ethernet flow spec to apply.
5001 * An Ethernet flow mask to apply.
5004 * 0 on success, a negative errno value otherwise and rte_errno is set.
5007 mlx5_ctrl_flow(struct rte_eth_dev *dev,
5008 struct rte_flow_item_eth *eth_spec,
5009 struct rte_flow_item_eth *eth_mask)
5011 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
5015 * Create default miss flow rule matching lacp traffic
5018 * Pointer to Ethernet device.
5020 * An Ethernet flow spec to apply.
5023 * 0 on success, a negative errno value otherwise and rte_errno is set.
5026 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
5028 struct mlx5_priv *priv = dev->data->dev_private;
5030 * The LACP matching is done by only using ether type since using
5031 * a multicast dst mac causes kernel to give low priority to this flow.
5033 static const struct rte_flow_item_eth lacp_spec = {
5034 .type = RTE_BE16(0x8809),
5036 static const struct rte_flow_item_eth lacp_mask = {
5039 const struct rte_flow_attr attr = {
5042 struct rte_flow_item items[] = {
5044 .type = RTE_FLOW_ITEM_TYPE_ETH,
5049 .type = RTE_FLOW_ITEM_TYPE_END,
5052 struct rte_flow_action actions[] = {
5054 .type = (enum rte_flow_action_type)
5055 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
5058 .type = RTE_FLOW_ACTION_TYPE_END,
5061 struct rte_flow_error error;
5062 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
5063 &attr, items, actions, false, &error);
5073 * @see rte_flow_destroy()
5077 mlx5_flow_destroy(struct rte_eth_dev *dev,
5078 struct rte_flow *flow,
5079 struct rte_flow_error *error __rte_unused)
5081 struct mlx5_priv *priv = dev->data->dev_private;
5083 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
5088 * Destroy all flows.
5090 * @see rte_flow_flush()
5094 mlx5_flow_flush(struct rte_eth_dev *dev,
5095 struct rte_flow_error *error __rte_unused)
5097 struct mlx5_priv *priv = dev->data->dev_private;
5099 mlx5_flow_list_flush(dev, &priv->flows, false);
5106 * @see rte_flow_isolate()
5110 mlx5_flow_isolate(struct rte_eth_dev *dev,
5112 struct rte_flow_error *error)
5114 struct mlx5_priv *priv = dev->data->dev_private;
5116 if (dev->data->dev_started) {
5117 rte_flow_error_set(error, EBUSY,
5118 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5120 "port must be stopped first");
5123 priv->isolated = !!enable;
5125 dev->dev_ops = &mlx5_os_dev_ops_isolate;
5127 dev->dev_ops = &mlx5_os_dev_ops;
5134 * @see rte_flow_query()
5138 flow_drv_query(struct rte_eth_dev *dev,
5140 const struct rte_flow_action *actions,
5142 struct rte_flow_error *error)
5144 struct mlx5_priv *priv = dev->data->dev_private;
5145 const struct mlx5_flow_driver_ops *fops;
5146 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5147 [MLX5_IPOOL_RTE_FLOW],
5149 enum mlx5_flow_drv_type ftype;
5152 return rte_flow_error_set(error, ENOENT,
5153 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5155 "invalid flow handle");
5157 ftype = flow->drv_type;
5158 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
5159 fops = flow_get_drv_ops(ftype);
5161 return fops->query(dev, flow, actions, data, error);
5167 * @see rte_flow_query()
5171 mlx5_flow_query(struct rte_eth_dev *dev,
5172 struct rte_flow *flow,
5173 const struct rte_flow_action *actions,
5175 struct rte_flow_error *error)
5179 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
5187 * Convert a flow director filter to a generic flow.
5190 * Pointer to Ethernet device.
5191 * @param fdir_filter
5192 * Flow director filter to add.
5194 * Generic flow parameters structure.
5197 * 0 on success, a negative errno value otherwise and rte_errno is set.
5200 flow_fdir_filter_convert(struct rte_eth_dev *dev,
5201 const struct rte_eth_fdir_filter *fdir_filter,
5202 struct mlx5_fdir *attributes)
5204 struct mlx5_priv *priv = dev->data->dev_private;
5205 const struct rte_eth_fdir_input *input = &fdir_filter->input;
5206 const struct rte_eth_fdir_masks *mask =
5207 &dev->data->dev_conf.fdir_conf.mask;
5209 /* Validate queue number. */
5210 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
5211 DRV_LOG(ERR, "port %u invalid queue number %d",
5212 dev->data->port_id, fdir_filter->action.rx_queue);
5216 attributes->attr.ingress = 1;
5217 attributes->items[0] = (struct rte_flow_item) {
5218 .type = RTE_FLOW_ITEM_TYPE_ETH,
5219 .spec = &attributes->l2,
5220 .mask = &attributes->l2_mask,
5222 switch (fdir_filter->action.behavior) {
5223 case RTE_ETH_FDIR_ACCEPT:
5224 attributes->actions[0] = (struct rte_flow_action){
5225 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
5226 .conf = &attributes->queue,
5229 case RTE_ETH_FDIR_REJECT:
5230 attributes->actions[0] = (struct rte_flow_action){
5231 .type = RTE_FLOW_ACTION_TYPE_DROP,
5235 DRV_LOG(ERR, "port %u invalid behavior %d",
5237 fdir_filter->action.behavior);
5238 rte_errno = ENOTSUP;
5241 attributes->queue.index = fdir_filter->action.rx_queue;
5243 switch (fdir_filter->input.flow_type) {
5244 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
5245 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
5246 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
5247 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){
5248 .src_addr = input->flow.ip4_flow.src_ip,
5249 .dst_addr = input->flow.ip4_flow.dst_ip,
5250 .time_to_live = input->flow.ip4_flow.ttl,
5251 .type_of_service = input->flow.ip4_flow.tos,
5253 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){
5254 .src_addr = mask->ipv4_mask.src_ip,
5255 .dst_addr = mask->ipv4_mask.dst_ip,
5256 .time_to_live = mask->ipv4_mask.ttl,
5257 .type_of_service = mask->ipv4_mask.tos,
5258 .next_proto_id = mask->ipv4_mask.proto,
5260 attributes->items[1] = (struct rte_flow_item){
5261 .type = RTE_FLOW_ITEM_TYPE_IPV4,
5262 .spec = &attributes->l3,
5263 .mask = &attributes->l3_mask,
5266 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
5267 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
5268 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
5269 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){
5270 .hop_limits = input->flow.ipv6_flow.hop_limits,
5271 .proto = input->flow.ipv6_flow.proto,
5274 memcpy(attributes->l3.ipv6.hdr.src_addr,
5275 input->flow.ipv6_flow.src_ip,
5276 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
5277 memcpy(attributes->l3.ipv6.hdr.dst_addr,
5278 input->flow.ipv6_flow.dst_ip,
5279 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
5280 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
5281 mask->ipv6_mask.src_ip,
5282 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
5283 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
5284 mask->ipv6_mask.dst_ip,
5285 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
5286 attributes->items[1] = (struct rte_flow_item){
5287 .type = RTE_FLOW_ITEM_TYPE_IPV6,
5288 .spec = &attributes->l3,
5289 .mask = &attributes->l3_mask,
5293 DRV_LOG(ERR, "port %u invalid flow type%d",
5294 dev->data->port_id, fdir_filter->input.flow_type);
5295 rte_errno = ENOTSUP;
5299 switch (fdir_filter->input.flow_type) {
5300 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
5301 attributes->l4.udp.hdr = (struct rte_udp_hdr){
5302 .src_port = input->flow.udp4_flow.src_port,
5303 .dst_port = input->flow.udp4_flow.dst_port,
5305 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
5306 .src_port = mask->src_port_mask,
5307 .dst_port = mask->dst_port_mask,
5309 attributes->items[2] = (struct rte_flow_item){
5310 .type = RTE_FLOW_ITEM_TYPE_UDP,
5311 .spec = &attributes->l4,
5312 .mask = &attributes->l4_mask,
5315 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
5316 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
5317 .src_port = input->flow.tcp4_flow.src_port,
5318 .dst_port = input->flow.tcp4_flow.dst_port,
5320 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
5321 .src_port = mask->src_port_mask,
5322 .dst_port = mask->dst_port_mask,
5324 attributes->items[2] = (struct rte_flow_item){
5325 .type = RTE_FLOW_ITEM_TYPE_TCP,
5326 .spec = &attributes->l4,
5327 .mask = &attributes->l4_mask,
5330 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
5331 attributes->l4.udp.hdr = (struct rte_udp_hdr){
5332 .src_port = input->flow.udp6_flow.src_port,
5333 .dst_port = input->flow.udp6_flow.dst_port,
5335 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
5336 .src_port = mask->src_port_mask,
5337 .dst_port = mask->dst_port_mask,
5339 attributes->items[2] = (struct rte_flow_item){
5340 .type = RTE_FLOW_ITEM_TYPE_UDP,
5341 .spec = &attributes->l4,
5342 .mask = &attributes->l4_mask,
5345 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
5346 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
5347 .src_port = input->flow.tcp6_flow.src_port,
5348 .dst_port = input->flow.tcp6_flow.dst_port,
5350 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
5351 .src_port = mask->src_port_mask,
5352 .dst_port = mask->dst_port_mask,
5354 attributes->items[2] = (struct rte_flow_item){
5355 .type = RTE_FLOW_ITEM_TYPE_TCP,
5356 .spec = &attributes->l4,
5357 .mask = &attributes->l4_mask,
5360 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
5361 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
5364 DRV_LOG(ERR, "port %u invalid flow type%d",
5365 dev->data->port_id, fdir_filter->input.flow_type);
5366 rte_errno = ENOTSUP;
5372 #define FLOW_FDIR_CMP(f1, f2, fld) \
5373 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
5376 * Compare two FDIR flows. If items and actions are identical, the two flows are
5380 * Pointer to Ethernet device.
5382 * FDIR flow to compare.
5384 * FDIR flow to compare.
5387 * Zero on match, 1 otherwise.
5390 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
5392 if (FLOW_FDIR_CMP(f1, f2, attr) ||
5393 FLOW_FDIR_CMP(f1, f2, l2) ||
5394 FLOW_FDIR_CMP(f1, f2, l2_mask) ||
5395 FLOW_FDIR_CMP(f1, f2, l3) ||
5396 FLOW_FDIR_CMP(f1, f2, l3_mask) ||
5397 FLOW_FDIR_CMP(f1, f2, l4) ||
5398 FLOW_FDIR_CMP(f1, f2, l4_mask) ||
5399 FLOW_FDIR_CMP(f1, f2, actions[0].type))
5401 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
5402 FLOW_FDIR_CMP(f1, f2, queue))
5408 * Search device flow list to find out a matched FDIR flow.
5411 * Pointer to Ethernet device.
5413 * FDIR flow to lookup.
5416 * Index of flow if found, 0 otherwise.
5419 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
5421 struct mlx5_priv *priv = dev->data->dev_private;
5422 uint32_t flow_idx = 0;
5423 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5425 MLX5_ASSERT(fdir_flow);
5426 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5427 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) {
5428 DRV_LOG(DEBUG, "port %u found FDIR flow %u",
5429 dev->data->port_id, flow_idx);
5430 flow_idx = priv_fdir_flow->rix_flow;
5438 * Add new flow director filter and store it in list.
5441 * Pointer to Ethernet device.
5442 * @param fdir_filter
5443 * Flow director filter to add.
5446 * 0 on success, a negative errno value otherwise and rte_errno is set.
5449 flow_fdir_filter_add(struct rte_eth_dev *dev,
5450 const struct rte_eth_fdir_filter *fdir_filter)
5452 struct mlx5_priv *priv = dev->data->dev_private;
5453 struct mlx5_fdir *fdir_flow;
5454 struct rte_flow *flow;
5455 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5459 fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0,
5465 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
5468 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow);
5473 priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO,
5474 sizeof(struct mlx5_fdir_flow),
5476 if (!priv_fdir_flow) {
5480 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
5481 fdir_flow->items, fdir_flow->actions, true,
5483 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5487 priv_fdir_flow->fdir = fdir_flow;
5488 priv_fdir_flow->rix_flow = flow_idx;
5489 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next);
5490 DRV_LOG(DEBUG, "port %u created FDIR flow %p",
5491 dev->data->port_id, (void *)flow);
5494 mlx5_free(priv_fdir_flow);
5495 mlx5_free(fdir_flow);
5500 * Delete specific filter.
5503 * Pointer to Ethernet device.
5504 * @param fdir_filter
5505 * Filter to be deleted.
5508 * 0 on success, a negative errno value otherwise and rte_errno is set.
5511 flow_fdir_filter_delete(struct rte_eth_dev *dev,
5512 const struct rte_eth_fdir_filter *fdir_filter)
5514 struct mlx5_priv *priv = dev->data->dev_private;
5516 struct mlx5_fdir fdir_flow = {
5519 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5522 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
5525 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5526 /* Find the fdir in priv list */
5527 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow))
5530 if (!priv_fdir_flow)
5532 LIST_REMOVE(priv_fdir_flow, next);
5533 flow_idx = priv_fdir_flow->rix_flow;
5534 flow_list_destroy(dev, &priv->flows, flow_idx);
5535 mlx5_free(priv_fdir_flow->fdir);
5536 mlx5_free(priv_fdir_flow);
5537 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u",
5538 dev->data->port_id, flow_idx);
5543 * Update queue for specific filter.
5546 * Pointer to Ethernet device.
5547 * @param fdir_filter
5548 * Filter to be updated.
5551 * 0 on success, a negative errno value otherwise and rte_errno is set.
5554 flow_fdir_filter_update(struct rte_eth_dev *dev,
5555 const struct rte_eth_fdir_filter *fdir_filter)
5559 ret = flow_fdir_filter_delete(dev, fdir_filter);
5562 return flow_fdir_filter_add(dev, fdir_filter);
5566 * Flush all filters.
5569 * Pointer to Ethernet device.
5572 flow_fdir_filter_flush(struct rte_eth_dev *dev)
5574 struct mlx5_priv *priv = dev->data->dev_private;
5575 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5577 while (!LIST_EMPTY(&priv->fdir_flows)) {
5578 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows);
5579 LIST_REMOVE(priv_fdir_flow, next);
5580 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow);
5581 mlx5_free(priv_fdir_flow->fdir);
5582 mlx5_free(priv_fdir_flow);
5587 * Get flow director information.
5590 * Pointer to Ethernet device.
5591 * @param[out] fdir_info
5592 * Resulting flow director information.
5595 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
5597 struct rte_eth_fdir_masks *mask =
5598 &dev->data->dev_conf.fdir_conf.mask;
5600 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
5601 fdir_info->guarant_spc = 0;
5602 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
5603 fdir_info->max_flexpayload = 0;
5604 fdir_info->flow_types_mask[0] = 0;
5605 fdir_info->flex_payload_unit = 0;
5606 fdir_info->max_flex_payload_segment_num = 0;
5607 fdir_info->flex_payload_limit = 0;
5608 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
5612 * Deal with flow director operations.
5615 * Pointer to Ethernet device.
5617 * Operation to perform.
5619 * Pointer to operation-specific structure.
5622 * 0 on success, a negative errno value otherwise and rte_errno is set.
5625 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
5628 enum rte_fdir_mode fdir_mode =
5629 dev->data->dev_conf.fdir_conf.mode;
5631 if (filter_op == RTE_ETH_FILTER_NOP)
5633 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
5634 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
5635 DRV_LOG(ERR, "port %u flow director mode %d not supported",
5636 dev->data->port_id, fdir_mode);
5640 switch (filter_op) {
5641 case RTE_ETH_FILTER_ADD:
5642 return flow_fdir_filter_add(dev, arg);
5643 case RTE_ETH_FILTER_UPDATE:
5644 return flow_fdir_filter_update(dev, arg);
5645 case RTE_ETH_FILTER_DELETE:
5646 return flow_fdir_filter_delete(dev, arg);
5647 case RTE_ETH_FILTER_FLUSH:
5648 flow_fdir_filter_flush(dev);
5650 case RTE_ETH_FILTER_INFO:
5651 flow_fdir_info_get(dev, arg);
5654 DRV_LOG(DEBUG, "port %u unknown operation %u",
5655 dev->data->port_id, filter_op);
5663 * Manage filter operations.
5666 * Pointer to Ethernet device structure.
5667 * @param filter_type
5670 * Operation to perform.
5672 * Pointer to operation-specific structure.
5675 * 0 on success, a negative errno value otherwise and rte_errno is set.
5678 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
5679 enum rte_filter_type filter_type,
5680 enum rte_filter_op filter_op,
5683 switch (filter_type) {
5684 case RTE_ETH_FILTER_GENERIC:
5685 if (filter_op != RTE_ETH_FILTER_GET) {
5689 *(const void **)arg = &mlx5_flow_ops;
5691 case RTE_ETH_FILTER_FDIR:
5692 return flow_fdir_ctrl_func(dev, filter_op, arg);
5694 DRV_LOG(ERR, "port %u filter type (%d) not supported",
5695 dev->data->port_id, filter_type);
5696 rte_errno = ENOTSUP;
5703 * Create the needed meter and suffix tables.
5706 * Pointer to Ethernet device.
5708 * Pointer to the flow meter.
5711 * Pointer to table set on success, NULL otherwise.
5713 struct mlx5_meter_domains_infos *
5714 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
5715 const struct mlx5_flow_meter *fm)
5717 const struct mlx5_flow_driver_ops *fops;
5719 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5720 return fops->create_mtr_tbls(dev, fm);
5724 * Destroy the meter table set.
5727 * Pointer to Ethernet device.
5729 * Pointer to the meter table set.
5735 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
5736 struct mlx5_meter_domains_infos *tbls)
5738 const struct mlx5_flow_driver_ops *fops;
5740 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5741 return fops->destroy_mtr_tbls(dev, tbls);
5745 * Create policer rules.
5748 * Pointer to Ethernet device.
5750 * Pointer to flow meter structure.
5752 * Pointer to flow attributes.
5755 * 0 on success, -1 otherwise.
5758 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
5759 struct mlx5_flow_meter *fm,
5760 const struct rte_flow_attr *attr)
5762 const struct mlx5_flow_driver_ops *fops;
5764 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5765 return fops->create_policer_rules(dev, fm, attr);
5769 * Destroy policer rules.
5772 * Pointer to flow meter structure.
5774 * Pointer to flow attributes.
5777 * 0 on success, -1 otherwise.
5780 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
5781 struct mlx5_flow_meter *fm,
5782 const struct rte_flow_attr *attr)
5784 const struct mlx5_flow_driver_ops *fops;
5786 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5787 return fops->destroy_policer_rules(dev, fm, attr);
5791 * Allocate a counter.
5794 * Pointer to Ethernet device structure.
5797 * Index to allocated counter on success, 0 otherwise.
5800 mlx5_counter_alloc(struct rte_eth_dev *dev)
5802 const struct mlx5_flow_driver_ops *fops;
5803 struct rte_flow_attr attr = { .transfer = 0 };
5805 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5806 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5807 return fops->counter_alloc(dev);
5810 "port %u counter allocate is not supported.",
5811 dev->data->port_id);
5819 * Pointer to Ethernet device structure.
5821 * Index to counter to be free.
5824 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
5826 const struct mlx5_flow_driver_ops *fops;
5827 struct rte_flow_attr attr = { .transfer = 0 };
5829 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5830 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5831 fops->counter_free(dev, cnt);
5835 "port %u counter free is not supported.",
5836 dev->data->port_id);
5840 * Query counter statistics.
5843 * Pointer to Ethernet device structure.
5845 * Index to counter to query.
5847 * Set to clear counter statistics.
5849 * The counter hits packets number to save.
5851 * The counter hits bytes number to save.
5854 * 0 on success, a negative errno value otherwise.
5857 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
5858 bool clear, uint64_t *pkts, uint64_t *bytes)
5860 const struct mlx5_flow_driver_ops *fops;
5861 struct rte_flow_attr attr = { .transfer = 0 };
5863 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5864 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5865 return fops->counter_query(dev, cnt, clear, pkts, bytes);
5868 "port %u counter query is not supported.",
5869 dev->data->port_id);
5873 #define MLX5_POOL_QUERY_FREQ_US 1000000
5876 * Get number of all validate pools.
5879 * Pointer to mlx5_dev_ctx_shared object.
5882 * The number of all validate pools.
5885 mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh)
5888 uint32_t pools_n = 0;
5890 for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i)
5891 pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid);
5896 * Set the periodic procedure for triggering asynchronous batch queries for all
5897 * the counter pools.
5900 * Pointer to mlx5_dev_ctx_shared object.
5903 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
5905 uint32_t pools_n, us;
5907 pools_n = mlx5_get_all_valid_pool_count(sh);
5908 us = MLX5_POOL_QUERY_FREQ_US / pools_n;
5909 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
5910 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
5911 sh->cmng.query_thread_on = 0;
5912 DRV_LOG(ERR, "Cannot reinitialize query alarm");
5914 sh->cmng.query_thread_on = 1;
5919 * The periodic procedure for triggering asynchronous batch queries for all the
5920 * counter pools. This function is probably called by the host thread.
5923 * The parameter for the alarm process.
5926 mlx5_flow_query_alarm(void *arg)
5928 struct mlx5_dev_ctx_shared *sh = arg;
5929 struct mlx5_devx_obj *dcs;
5932 uint8_t batch = sh->cmng.batch;
5933 uint8_t age = sh->cmng.age;
5934 uint16_t pool_index = sh->cmng.pool_index;
5935 struct mlx5_pools_container *cont;
5936 struct mlx5_flow_counter_pool *pool;
5937 int cont_loop = MLX5_CCONT_TYPE_MAX;
5939 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
5942 cont = MLX5_CNT_CONTAINER(sh, batch, age);
5943 rte_spinlock_lock(&cont->resize_sl);
5945 rte_spinlock_unlock(&cont->resize_sl);
5946 /* Check if all the containers are empty. */
5947 if (unlikely(--cont_loop == 0))
5951 if (batch == 0 && pool_index == 0) {
5953 sh->cmng.batch = batch;
5956 goto next_container;
5958 pool = cont->pools[pool_index];
5959 rte_spinlock_unlock(&cont->resize_sl);
5961 /* There is a pool query in progress. */
5964 LIST_FIRST(&sh->cmng.free_stat_raws);
5966 /* No free counter statistics raw memory. */
5968 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read
5970 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL;
5972 * Identify the counters released between query trigger and query
5973 * handle more effiecntly. The counter released in this gap period
5974 * should wait for a new round of query as the new arrived packets
5975 * will not be taken into account.
5978 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL -
5980 pool->raw_hw->mem_mng->dm->id,
5982 (pool->raw_hw->data + offset),
5984 (uint64_t)(uintptr_t)pool);
5986 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
5987 " %d", pool->min_dcs->id);
5988 pool->raw_hw = NULL;
5991 pool->raw_hw->min_dcs_id = dcs->id;
5992 LIST_REMOVE(pool->raw_hw, next);
5993 sh->cmng.pending_queries++;
5995 if (pool_index >= rte_atomic16_read(&cont->n_valid)) {
5998 if (batch == 0 && pool_index == 0)
6002 sh->cmng.batch = batch;
6003 sh->cmng.pool_index = pool_index;
6005 mlx5_set_query_alarm(sh);
6009 * Check and callback event for new aged flow in the counter pool
6012 * Pointer to mlx5_dev_ctx_shared object.
6014 * Pointer to Current counter pool.
6017 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
6018 struct mlx5_flow_counter_pool *pool)
6020 struct mlx5_priv *priv;
6021 struct mlx5_flow_counter *cnt;
6022 struct mlx5_age_info *age_info;
6023 struct mlx5_age_param *age_param;
6024 struct mlx5_counter_stats_raw *cur = pool->raw_hw;
6025 struct mlx5_counter_stats_raw *prev = pool->raw;
6026 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10);
6029 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
6030 cnt = MLX5_POOL_GET_CNT(pool, i);
6031 age_param = MLX5_CNT_TO_AGE(cnt);
6032 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE)
6034 if (cur->data[i].hits != prev->data[i].hits) {
6035 age_param->expire = curr + age_param->timeout;
6038 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2))
6041 * Hold the lock first, or if between the
6042 * state AGE_TMOUT and tailq operation the
6043 * release happened, the release procedure
6044 * may delete a non-existent tailq node.
6046 priv = rte_eth_devices[age_param->port_id].data->dev_private;
6047 age_info = GET_PORT_AGE_INFO(priv);
6048 rte_spinlock_lock(&age_info->aged_sl);
6049 /* If the cpmset fails, release happens. */
6050 if (rte_atomic16_cmpset((volatile uint16_t *)
6055 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
6056 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
6058 rte_spinlock_unlock(&age_info->aged_sl);
6060 for (i = 0; i < sh->max_port; i++) {
6061 age_info = &sh->port[i].age_info;
6062 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW))
6064 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER))
6065 _rte_eth_dev_callback_process
6066 (&rte_eth_devices[sh->port[i].devx_ih_port_id],
6067 RTE_ETH_EVENT_FLOW_AGED, NULL);
6068 age_info->flags = 0;
6073 * Handler for the HW respond about ready values from an asynchronous batch
6074 * query. This function is probably called by the host thread.
6077 * The pointer to the shared device context.
6078 * @param[in] async_id
6079 * The Devx async ID.
6081 * The status of the completion.
6084 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
6085 uint64_t async_id, int status)
6087 struct mlx5_flow_counter_pool *pool =
6088 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6089 struct mlx5_counter_stats_raw *raw_to_free;
6090 uint8_t age = !!IS_AGE_POOL(pool);
6091 uint8_t query_gen = pool->query_gen ^ 1;
6092 struct mlx5_pools_container *cont =
6093 MLX5_CNT_CONTAINER(sh, !IS_EXT_POOL(pool), age);
6095 if (unlikely(status)) {
6096 raw_to_free = pool->raw_hw;
6098 raw_to_free = pool->raw;
6099 if (IS_AGE_POOL(pool))
6100 mlx5_flow_aging_check(sh, pool);
6101 rte_spinlock_lock(&pool->sl);
6102 pool->raw = pool->raw_hw;
6103 rte_spinlock_unlock(&pool->sl);
6104 /* Be sure the new raw counters data is updated in memory. */
6106 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
6107 rte_spinlock_lock(&cont->csl);
6108 TAILQ_CONCAT(&cont->counters,
6109 &pool->counters[query_gen], next);
6110 rte_spinlock_unlock(&cont->csl);
6113 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6114 pool->raw_hw = NULL;
6115 sh->cmng.pending_queries--;
6119 * Translate the rte_flow group index to HW table value.
6121 * @param[in] attributes
6122 * Pointer to flow attributes
6123 * @param[in] external
6124 * Value is part of flow rule created by request external to PMD.
6126 * rte_flow group index value.
6127 * @param[out] fdb_def_rule
6128 * Whether fdb jump to table 1 is configured.
6132 * Pointer to error structure.
6135 * 0 on success, a negative errno value otherwise and rte_errno is set.
6138 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external,
6139 uint32_t group, bool fdb_def_rule, uint32_t *table,
6140 struct rte_flow_error *error)
6142 if (attributes->transfer && external && fdb_def_rule) {
6143 if (group == UINT32_MAX)
6144 return rte_flow_error_set
6146 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6148 "group index not supported");
6157 * Discover availability of metadata reg_c's.
6159 * Iteratively use test flows to check availability.
6162 * Pointer to the Ethernet device structure.
6165 * 0 on success, a negative errno value otherwise and rte_errno is set.
6168 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6170 struct mlx5_priv *priv = dev->data->dev_private;
6171 struct mlx5_dev_config *config = &priv->config;
6172 enum modify_reg idx;
6175 /* reg_c[0] and reg_c[1] are reserved. */
6176 config->flow_mreg_c[n++] = REG_C_0;
6177 config->flow_mreg_c[n++] = REG_C_1;
6178 /* Discover availability of other reg_c's. */
6179 for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6180 struct rte_flow_attr attr = {
6181 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6182 .priority = MLX5_FLOW_PRIO_RSVD,
6185 struct rte_flow_item items[] = {
6187 .type = RTE_FLOW_ITEM_TYPE_END,
6190 struct rte_flow_action actions[] = {
6192 .type = (enum rte_flow_action_type)
6193 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6194 .conf = &(struct mlx5_flow_action_copy_mreg){
6200 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6201 .conf = &(struct rte_flow_action_jump){
6202 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6206 .type = RTE_FLOW_ACTION_TYPE_END,
6210 struct rte_flow *flow;
6211 struct rte_flow_error error;
6213 if (!config->dv_flow_en)
6215 /* Create internal flow, validation skips copy action. */
6216 flow_idx = flow_list_create(dev, NULL, &attr, items,
6217 actions, false, &error);
6218 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6222 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
6223 config->flow_mreg_c[n++] = idx;
6224 flow_list_destroy(dev, NULL, flow_idx);
6226 for (; n < MLX5_MREG_C_NUM; ++n)
6227 config->flow_mreg_c[n] = REG_NONE;
6232 * Dump flow raw hw data to file
6235 * The pointer to Ethernet device.
6237 * A pointer to a file for output.
6239 * Perform verbose error reporting if not NULL. PMDs initialize this
6240 * structure in case of error only.
6242 * 0 on success, a nagative value otherwise.
6245 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6247 struct rte_flow_error *error __rte_unused)
6249 struct mlx5_priv *priv = dev->data->dev_private;
6250 struct mlx5_dev_ctx_shared *sh = priv->sh;
6252 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6253 sh->tx_domain, file);
6257 * Get aged-out flows.
6260 * Pointer to the Ethernet device structure.
6261 * @param[in] context
6262 * The address of an array of pointers to the aged-out flows contexts.
6263 * @param[in] nb_countexts
6264 * The length of context array pointers.
6266 * Perform verbose error reporting if not NULL. Initialized in case of
6270 * how many contexts get in success, otherwise negative errno value.
6271 * if nb_contexts is 0, return the amount of all aged contexts.
6272 * if nb_contexts is not 0 , return the amount of aged flows reported
6273 * in the context array.
6276 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6277 uint32_t nb_contexts, struct rte_flow_error *error)
6279 const struct mlx5_flow_driver_ops *fops;
6280 struct rte_flow_attr attr = { .transfer = 0 };
6282 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6283 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6284 return fops->get_aged_flows(dev, contexts, nb_contexts,
6288 "port %u get aged flows is not supported.",
6289 dev->data->port_id);