1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox Technologies, Ltd
6 #include <netinet/in.h>
14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
16 #pragma GCC diagnostic ignored "-Wpedantic"
18 #include <infiniband/verbs.h>
20 #pragma GCC diagnostic error "-Wpedantic"
23 #include <rte_common.h>
24 #include <rte_ether.h>
25 #include <rte_ethdev_driver.h>
27 #include <rte_cycles.h>
28 #include <rte_flow_driver.h>
29 #include <rte_malloc.h>
32 #include <mlx5_glue.h>
33 #include <mlx5_devx_cmds.h>
36 #include "mlx5_defs.h"
38 #include "mlx5_flow.h"
39 #include "mlx5_rxtx.h"
41 /* Dev ops structure defined in mlx5.c */
42 extern const struct eth_dev_ops mlx5_dev_ops;
43 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
45 /** Device flow drivers. */
46 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
47 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops;
49 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
51 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
53 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
54 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
55 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
56 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
58 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
59 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
64 MLX5_EXPANSION_ROOT_OUTER,
65 MLX5_EXPANSION_ROOT_ETH_VLAN,
66 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
67 MLX5_EXPANSION_OUTER_ETH,
68 MLX5_EXPANSION_OUTER_ETH_VLAN,
69 MLX5_EXPANSION_OUTER_VLAN,
70 MLX5_EXPANSION_OUTER_IPV4,
71 MLX5_EXPANSION_OUTER_IPV4_UDP,
72 MLX5_EXPANSION_OUTER_IPV4_TCP,
73 MLX5_EXPANSION_OUTER_IPV6,
74 MLX5_EXPANSION_OUTER_IPV6_UDP,
75 MLX5_EXPANSION_OUTER_IPV6_TCP,
77 MLX5_EXPANSION_VXLAN_GPE,
81 MLX5_EXPANSION_ETH_VLAN,
84 MLX5_EXPANSION_IPV4_UDP,
85 MLX5_EXPANSION_IPV4_TCP,
87 MLX5_EXPANSION_IPV6_UDP,
88 MLX5_EXPANSION_IPV6_TCP,
91 /** Supported expansion of items. */
92 static const struct rte_flow_expand_node mlx5_support_expansion[] = {
93 [MLX5_EXPANSION_ROOT] = {
94 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
97 .type = RTE_FLOW_ITEM_TYPE_END,
99 [MLX5_EXPANSION_ROOT_OUTER] = {
100 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
101 MLX5_EXPANSION_OUTER_IPV4,
102 MLX5_EXPANSION_OUTER_IPV6),
103 .type = RTE_FLOW_ITEM_TYPE_END,
105 [MLX5_EXPANSION_ROOT_ETH_VLAN] = {
106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
107 .type = RTE_FLOW_ITEM_TYPE_END,
109 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
111 .type = RTE_FLOW_ITEM_TYPE_END,
113 [MLX5_EXPANSION_OUTER_ETH] = {
114 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
115 MLX5_EXPANSION_OUTER_IPV6,
116 MLX5_EXPANSION_MPLS),
117 .type = RTE_FLOW_ITEM_TYPE_ETH,
120 [MLX5_EXPANSION_OUTER_ETH_VLAN] = {
121 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
122 .type = RTE_FLOW_ITEM_TYPE_ETH,
125 [MLX5_EXPANSION_OUTER_VLAN] = {
126 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
127 MLX5_EXPANSION_OUTER_IPV6),
128 .type = RTE_FLOW_ITEM_TYPE_VLAN,
130 [MLX5_EXPANSION_OUTER_IPV4] = {
131 .next = RTE_FLOW_EXPAND_RSS_NEXT
132 (MLX5_EXPANSION_OUTER_IPV4_UDP,
133 MLX5_EXPANSION_OUTER_IPV4_TCP,
136 MLX5_EXPANSION_IPV6),
137 .type = RTE_FLOW_ITEM_TYPE_IPV4,
138 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
139 ETH_RSS_NONFRAG_IPV4_OTHER,
141 [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
142 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
143 MLX5_EXPANSION_VXLAN_GPE),
144 .type = RTE_FLOW_ITEM_TYPE_UDP,
145 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
147 [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
148 .type = RTE_FLOW_ITEM_TYPE_TCP,
149 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
151 [MLX5_EXPANSION_OUTER_IPV6] = {
152 .next = RTE_FLOW_EXPAND_RSS_NEXT
153 (MLX5_EXPANSION_OUTER_IPV6_UDP,
154 MLX5_EXPANSION_OUTER_IPV6_TCP,
156 MLX5_EXPANSION_IPV6),
157 .type = RTE_FLOW_ITEM_TYPE_IPV6,
158 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
159 ETH_RSS_NONFRAG_IPV6_OTHER,
161 [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
162 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
163 MLX5_EXPANSION_VXLAN_GPE),
164 .type = RTE_FLOW_ITEM_TYPE_UDP,
165 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
167 [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
168 .type = RTE_FLOW_ITEM_TYPE_TCP,
169 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
171 [MLX5_EXPANSION_VXLAN] = {
172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
174 MLX5_EXPANSION_IPV6),
175 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
177 [MLX5_EXPANSION_VXLAN_GPE] = {
178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
180 MLX5_EXPANSION_IPV6),
181 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
183 [MLX5_EXPANSION_GRE] = {
184 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
185 .type = RTE_FLOW_ITEM_TYPE_GRE,
187 [MLX5_EXPANSION_MPLS] = {
188 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
189 MLX5_EXPANSION_IPV6),
190 .type = RTE_FLOW_ITEM_TYPE_MPLS,
192 [MLX5_EXPANSION_ETH] = {
193 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
194 MLX5_EXPANSION_IPV6),
195 .type = RTE_FLOW_ITEM_TYPE_ETH,
197 [MLX5_EXPANSION_ETH_VLAN] = {
198 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
199 .type = RTE_FLOW_ITEM_TYPE_ETH,
201 [MLX5_EXPANSION_VLAN] = {
202 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
203 MLX5_EXPANSION_IPV6),
204 .type = RTE_FLOW_ITEM_TYPE_VLAN,
206 [MLX5_EXPANSION_IPV4] = {
207 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
208 MLX5_EXPANSION_IPV4_TCP),
209 .type = RTE_FLOW_ITEM_TYPE_IPV4,
210 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
211 ETH_RSS_NONFRAG_IPV4_OTHER,
213 [MLX5_EXPANSION_IPV4_UDP] = {
214 .type = RTE_FLOW_ITEM_TYPE_UDP,
215 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
217 [MLX5_EXPANSION_IPV4_TCP] = {
218 .type = RTE_FLOW_ITEM_TYPE_TCP,
219 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
221 [MLX5_EXPANSION_IPV6] = {
222 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
223 MLX5_EXPANSION_IPV6_TCP),
224 .type = RTE_FLOW_ITEM_TYPE_IPV6,
225 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
226 ETH_RSS_NONFRAG_IPV6_OTHER,
228 [MLX5_EXPANSION_IPV6_UDP] = {
229 .type = RTE_FLOW_ITEM_TYPE_UDP,
230 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
232 [MLX5_EXPANSION_IPV6_TCP] = {
233 .type = RTE_FLOW_ITEM_TYPE_TCP,
234 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
238 static const struct rte_flow_ops mlx5_flow_ops = {
239 .validate = mlx5_flow_validate,
240 .create = mlx5_flow_create,
241 .destroy = mlx5_flow_destroy,
242 .flush = mlx5_flow_flush,
243 .isolate = mlx5_flow_isolate,
244 .query = mlx5_flow_query,
245 .dev_dump = mlx5_flow_dev_dump,
246 .get_aged_flows = mlx5_flow_get_aged_flows,
249 /* Convert FDIR request to Generic flow. */
251 struct rte_flow_attr attr;
252 struct rte_flow_item items[4];
253 struct rte_flow_item_eth l2;
254 struct rte_flow_item_eth l2_mask;
256 struct rte_flow_item_ipv4 ipv4;
257 struct rte_flow_item_ipv6 ipv6;
260 struct rte_flow_item_ipv4 ipv4;
261 struct rte_flow_item_ipv6 ipv6;
264 struct rte_flow_item_udp udp;
265 struct rte_flow_item_tcp tcp;
268 struct rte_flow_item_udp udp;
269 struct rte_flow_item_tcp tcp;
271 struct rte_flow_action actions[2];
272 struct rte_flow_action_queue queue;
275 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
276 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
277 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
280 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
281 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
282 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
283 { 9, 10, 11 }, { 12, 13, 14 },
286 /* Tunnel information. */
287 struct mlx5_flow_tunnel_info {
288 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
289 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
292 static struct mlx5_flow_tunnel_info tunnels_info[] = {
294 .tunnel = MLX5_FLOW_LAYER_VXLAN,
295 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
298 .tunnel = MLX5_FLOW_LAYER_GENEVE,
299 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
302 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
303 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
306 .tunnel = MLX5_FLOW_LAYER_GRE,
307 .ptype = RTE_PTYPE_TUNNEL_GRE,
310 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
311 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
314 .tunnel = MLX5_FLOW_LAYER_MPLS,
315 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
318 .tunnel = MLX5_FLOW_LAYER_NVGRE,
319 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
322 .tunnel = MLX5_FLOW_LAYER_IPIP,
323 .ptype = RTE_PTYPE_TUNNEL_IP,
326 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
327 .ptype = RTE_PTYPE_TUNNEL_IP,
330 .tunnel = MLX5_FLOW_LAYER_GTP,
331 .ptype = RTE_PTYPE_TUNNEL_GTPU,
336 * Translate tag ID to register.
339 * Pointer to the Ethernet device structure.
341 * The feature that request the register.
343 * The request register ID.
345 * Error description in case of any.
348 * The request register on success, a negative errno
349 * value otherwise and rte_errno is set.
352 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
353 enum mlx5_feature_name feature,
355 struct rte_flow_error *error)
357 struct mlx5_priv *priv = dev->data->dev_private;
358 struct mlx5_dev_config *config = &priv->config;
359 enum modify_reg start_reg;
360 bool skip_mtr_reg = false;
363 case MLX5_HAIRPIN_RX:
365 case MLX5_HAIRPIN_TX:
367 case MLX5_METADATA_RX:
368 switch (config->dv_xmeta_en) {
369 case MLX5_XMETA_MODE_LEGACY:
371 case MLX5_XMETA_MODE_META16:
373 case MLX5_XMETA_MODE_META32:
377 case MLX5_METADATA_TX:
379 case MLX5_METADATA_FDB:
380 switch (config->dv_xmeta_en) {
381 case MLX5_XMETA_MODE_LEGACY:
383 case MLX5_XMETA_MODE_META16:
385 case MLX5_XMETA_MODE_META32:
390 switch (config->dv_xmeta_en) {
391 case MLX5_XMETA_MODE_LEGACY:
393 case MLX5_XMETA_MODE_META16:
395 case MLX5_XMETA_MODE_META32:
401 * If meter color and flow match share one register, flow match
402 * should use the meter color register for match.
404 if (priv->mtr_reg_share)
405 return priv->mtr_color_reg;
407 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
410 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE);
411 return priv->mtr_color_reg;
414 * Metadata COPY_MARK register using is in meter suffix sub
415 * flow while with meter. It's safe to share the same register.
417 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
420 * If meter is enable, it will engage the register for color
421 * match and flow match. If meter color match is not using the
422 * REG_C_2, need to skip the REG_C_x be used by meter color
424 * If meter is disable, free to use all available registers.
426 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
427 (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
428 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
429 if (id > (REG_C_7 - start_reg))
430 return rte_flow_error_set(error, EINVAL,
431 RTE_FLOW_ERROR_TYPE_ITEM,
432 NULL, "invalid tag id");
433 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE)
434 return rte_flow_error_set(error, ENOTSUP,
435 RTE_FLOW_ERROR_TYPE_ITEM,
436 NULL, "unsupported tag id");
438 * This case means meter is using the REG_C_x great than 2.
439 * Take care not to conflict with meter color REG_C_x.
440 * If the available index REG_C_y >= REG_C_x, skip the
443 if (skip_mtr_reg && config->flow_mreg_c
444 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
445 if (config->flow_mreg_c
446 [id + 1 + start_reg - REG_C_0] != REG_NONE)
447 return config->flow_mreg_c
448 [id + 1 + start_reg - REG_C_0];
449 return rte_flow_error_set(error, ENOTSUP,
450 RTE_FLOW_ERROR_TYPE_ITEM,
451 NULL, "unsupported tag id");
453 return config->flow_mreg_c[id + start_reg - REG_C_0];
456 return rte_flow_error_set(error, EINVAL,
457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
458 NULL, "invalid feature name");
462 * Check extensive flow metadata register support.
465 * Pointer to rte_eth_dev structure.
468 * True if device supports extensive flow metadata register, otherwise false.
471 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
473 struct mlx5_priv *priv = dev->data->dev_private;
474 struct mlx5_dev_config *config = &priv->config;
477 * Having available reg_c can be regarded inclusively as supporting
478 * extensive flow metadata register, which could mean,
479 * - metadata register copy action by modify header.
480 * - 16 modify header actions is supported.
481 * - reg_c's are preserved across different domain (FDB and NIC) on
482 * packet loopback by flow lookup miss.
484 return config->flow_mreg_c[2] != REG_NONE;
488 * Discover the maximum number of priority available.
491 * Pointer to the Ethernet device structure.
494 * number of supported flow priority on success, a negative errno
495 * value otherwise and rte_errno is set.
498 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
500 struct mlx5_priv *priv = dev->data->dev_private;
502 struct ibv_flow_attr attr;
503 struct ibv_flow_spec_eth eth;
504 struct ibv_flow_spec_action_drop drop;
508 .port = (uint8_t)priv->ibv_port,
511 .type = IBV_FLOW_SPEC_ETH,
512 .size = sizeof(struct ibv_flow_spec_eth),
515 .size = sizeof(struct ibv_flow_spec_action_drop),
516 .type = IBV_FLOW_SPEC_ACTION_DROP,
519 struct ibv_flow *flow;
520 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
521 uint16_t vprio[] = { 8, 16 };
529 for (i = 0; i != RTE_DIM(vprio); i++) {
530 flow_attr.attr.priority = vprio[i] - 1;
531 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
534 claim_zero(mlx5_glue->destroy_flow(flow));
537 mlx5_hrxq_drop_release(dev);
540 priority = RTE_DIM(priority_map_3);
543 priority = RTE_DIM(priority_map_5);
548 "port %u verbs maximum priority: %d expected 8/16",
549 dev->data->port_id, priority);
552 DRV_LOG(INFO, "port %u flow maximum priority: %d",
553 dev->data->port_id, priority);
558 * Adjust flow priority based on the highest layer and the request priority.
561 * Pointer to the Ethernet device structure.
562 * @param[in] priority
563 * The rule base priority.
564 * @param[in] subpriority
565 * The priority based on the items.
570 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
571 uint32_t subpriority)
574 struct mlx5_priv *priv = dev->data->dev_private;
576 switch (priv->config.flow_prio) {
577 case RTE_DIM(priority_map_3):
578 res = priority_map_3[priority][subpriority];
580 case RTE_DIM(priority_map_5):
581 res = priority_map_5[priority][subpriority];
588 * Verify the @p item specifications (spec, last, mask) are compatible with the
592 * Item specification.
594 * @p item->mask or flow default bit-masks.
595 * @param[in] nic_mask
596 * Bit-masks covering supported fields by the NIC to compare with user mask.
598 * Bit-masks size in bytes.
600 * Pointer to error structure.
603 * 0 on success, a negative errno value otherwise and rte_errno is set.
606 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
608 const uint8_t *nic_mask,
610 struct rte_flow_error *error)
614 MLX5_ASSERT(nic_mask);
615 for (i = 0; i < size; ++i)
616 if ((nic_mask[i] | mask[i]) != nic_mask[i])
617 return rte_flow_error_set(error, ENOTSUP,
618 RTE_FLOW_ERROR_TYPE_ITEM,
620 "mask enables non supported"
622 if (!item->spec && (item->mask || item->last))
623 return rte_flow_error_set(error, EINVAL,
624 RTE_FLOW_ERROR_TYPE_ITEM, item,
625 "mask/last without a spec is not"
627 if (item->spec && item->last) {
633 for (i = 0; i < size; ++i) {
634 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
635 last[i] = ((const uint8_t *)item->last)[i] & mask[i];
637 ret = memcmp(spec, last, size);
639 return rte_flow_error_set(error, EINVAL,
640 RTE_FLOW_ERROR_TYPE_ITEM,
642 "range is not valid");
648 * Adjust the hash fields according to the @p flow information.
650 * @param[in] dev_flow.
651 * Pointer to the mlx5_flow.
653 * 1 when the hash field is for a tunnel item.
654 * @param[in] layer_types
656 * @param[in] hash_fields
660 * The hash fields that should be used.
663 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
664 int tunnel __rte_unused, uint64_t layer_types,
665 uint64_t hash_fields)
667 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
668 int rss_request_inner = rss_desc->level >= 2;
670 /* Check RSS hash level for tunnel. */
671 if (tunnel && rss_request_inner)
672 hash_fields |= IBV_RX_HASH_INNER;
673 else if (tunnel || rss_request_inner)
676 /* Check if requested layer matches RSS hash fields. */
677 if (!(rss_desc->types & layer_types))
683 * Lookup and set the ptype in the data Rx part. A single Ptype can be used,
684 * if several tunnel rules are used on this queue, the tunnel ptype will be
688 * Rx queue to update.
691 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
694 uint32_t tunnel_ptype = 0;
696 /* Look up for the ptype to use. */
697 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
698 if (!rxq_ctrl->flow_tunnels_n[i])
701 tunnel_ptype = tunnels_info[i].ptype;
707 rxq_ctrl->rxq.tunnel = tunnel_ptype;
711 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
715 * Pointer to the Ethernet device structure.
716 * @param[in] dev_handle
717 * Pointer to device flow handle structure.
720 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
721 struct mlx5_flow_handle *dev_handle)
723 struct mlx5_priv *priv = dev->data->dev_private;
724 const int mark = dev_handle->mark;
725 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
726 struct mlx5_hrxq *hrxq;
729 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
731 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
732 dev_handle->rix_hrxq);
735 for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
736 int idx = hrxq->ind_table->queues[i];
737 struct mlx5_rxq_ctrl *rxq_ctrl =
738 container_of((*priv->rxqs)[idx],
739 struct mlx5_rxq_ctrl, rxq);
742 * To support metadata register copy on Tx loopback,
743 * this must be always enabled (metadata may arive
744 * from other port - not from local flows only.
746 if (priv->config.dv_flow_en &&
747 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
748 mlx5_flow_ext_mreg_supported(dev)) {
749 rxq_ctrl->rxq.mark = 1;
750 rxq_ctrl->flow_mark_n = 1;
752 rxq_ctrl->rxq.mark = 1;
753 rxq_ctrl->flow_mark_n++;
758 /* Increase the counter matching the flow. */
759 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
760 if ((tunnels_info[j].tunnel &
761 dev_handle->layers) ==
762 tunnels_info[j].tunnel) {
763 rxq_ctrl->flow_tunnels_n[j]++;
767 flow_rxq_tunnel_ptype_update(rxq_ctrl);
773 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
776 * Pointer to the Ethernet device structure.
778 * Pointer to flow structure.
781 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
783 struct mlx5_priv *priv = dev->data->dev_private;
785 struct mlx5_flow_handle *dev_handle;
787 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
788 handle_idx, dev_handle, next)
789 flow_drv_rxq_flags_set(dev, dev_handle);
793 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
794 * device flow if no other flow uses it with the same kind of request.
797 * Pointer to Ethernet device.
798 * @param[in] dev_handle
799 * Pointer to the device flow handle structure.
802 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
803 struct mlx5_flow_handle *dev_handle)
805 struct mlx5_priv *priv = dev->data->dev_private;
806 const int mark = dev_handle->mark;
807 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
808 struct mlx5_hrxq *hrxq;
811 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE)
813 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
814 dev_handle->rix_hrxq);
817 MLX5_ASSERT(dev->data->dev_started);
818 for (i = 0; i != hrxq->ind_table->queues_n; ++i) {
819 int idx = hrxq->ind_table->queues[i];
820 struct mlx5_rxq_ctrl *rxq_ctrl =
821 container_of((*priv->rxqs)[idx],
822 struct mlx5_rxq_ctrl, rxq);
824 if (priv->config.dv_flow_en &&
825 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
826 mlx5_flow_ext_mreg_supported(dev)) {
827 rxq_ctrl->rxq.mark = 1;
828 rxq_ctrl->flow_mark_n = 1;
830 rxq_ctrl->flow_mark_n--;
831 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
836 /* Decrease the counter matching the flow. */
837 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
838 if ((tunnels_info[j].tunnel &
839 dev_handle->layers) ==
840 tunnels_info[j].tunnel) {
841 rxq_ctrl->flow_tunnels_n[j]--;
845 flow_rxq_tunnel_ptype_update(rxq_ctrl);
851 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
852 * @p flow if no other flow uses it with the same kind of request.
855 * Pointer to Ethernet device.
857 * Pointer to the flow.
860 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
862 struct mlx5_priv *priv = dev->data->dev_private;
864 struct mlx5_flow_handle *dev_handle;
866 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
867 handle_idx, dev_handle, next)
868 flow_drv_rxq_flags_trim(dev, dev_handle);
872 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
875 * Pointer to Ethernet device.
878 flow_rxq_flags_clear(struct rte_eth_dev *dev)
880 struct mlx5_priv *priv = dev->data->dev_private;
883 for (i = 0; i != priv->rxqs_n; ++i) {
884 struct mlx5_rxq_ctrl *rxq_ctrl;
887 if (!(*priv->rxqs)[i])
889 rxq_ctrl = container_of((*priv->rxqs)[i],
890 struct mlx5_rxq_ctrl, rxq);
891 rxq_ctrl->flow_mark_n = 0;
892 rxq_ctrl->rxq.mark = 0;
893 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
894 rxq_ctrl->flow_tunnels_n[j] = 0;
895 rxq_ctrl->rxq.tunnel = 0;
900 * Set the Rx queue dynamic metadata (mask and offset) for a flow
903 * Pointer to the Ethernet device structure.
906 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
908 struct mlx5_priv *priv = dev->data->dev_private;
909 struct mlx5_rxq_data *data;
912 for (i = 0; i != priv->rxqs_n; ++i) {
913 if (!(*priv->rxqs)[i])
915 data = (*priv->rxqs)[i];
916 if (!rte_flow_dynf_metadata_avail()) {
918 data->flow_meta_mask = 0;
919 data->flow_meta_offset = -1;
922 data->flow_meta_mask = rte_flow_dynf_metadata_mask;
923 data->flow_meta_offset = rte_flow_dynf_metadata_offs;
929 * return a pointer to the desired action in the list of actions.
932 * The list of actions to search the action in.
934 * The action to find.
937 * Pointer to the action in the list, if found. NULL otherwise.
939 const struct rte_flow_action *
940 mlx5_flow_find_action(const struct rte_flow_action *actions,
941 enum rte_flow_action_type action)
945 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
946 if (actions->type == action)
952 * Validate the flag action.
954 * @param[in] action_flags
955 * Bit-fields that holds the actions detected until now.
957 * Attributes of flow that includes this action.
959 * Pointer to error structure.
962 * 0 on success, a negative errno value otherwise and rte_errno is set.
965 mlx5_flow_validate_action_flag(uint64_t action_flags,
966 const struct rte_flow_attr *attr,
967 struct rte_flow_error *error)
969 if (action_flags & MLX5_FLOW_ACTION_MARK)
970 return rte_flow_error_set(error, EINVAL,
971 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
972 "can't mark and flag in same flow");
973 if (action_flags & MLX5_FLOW_ACTION_FLAG)
974 return rte_flow_error_set(error, EINVAL,
975 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
977 " actions in same flow");
979 return rte_flow_error_set(error, ENOTSUP,
980 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
981 "flag action not supported for "
987 * Validate the mark action.
990 * Pointer to the queue action.
991 * @param[in] action_flags
992 * Bit-fields that holds the actions detected until now.
994 * Attributes of flow that includes this action.
996 * Pointer to error structure.
999 * 0 on success, a negative errno value otherwise and rte_errno is set.
1002 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1003 uint64_t action_flags,
1004 const struct rte_flow_attr *attr,
1005 struct rte_flow_error *error)
1007 const struct rte_flow_action_mark *mark = action->conf;
1010 return rte_flow_error_set(error, EINVAL,
1011 RTE_FLOW_ERROR_TYPE_ACTION,
1013 "configuration cannot be null");
1014 if (mark->id >= MLX5_FLOW_MARK_MAX)
1015 return rte_flow_error_set(error, EINVAL,
1016 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1018 "mark id must in 0 <= id < "
1019 RTE_STR(MLX5_FLOW_MARK_MAX));
1020 if (action_flags & MLX5_FLOW_ACTION_FLAG)
1021 return rte_flow_error_set(error, EINVAL,
1022 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1023 "can't flag and mark in same flow");
1024 if (action_flags & MLX5_FLOW_ACTION_MARK)
1025 return rte_flow_error_set(error, EINVAL,
1026 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1027 "can't have 2 mark actions in same"
1030 return rte_flow_error_set(error, ENOTSUP,
1031 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1032 "mark action not supported for "
1038 * Validate the drop action.
1040 * @param[in] action_flags
1041 * Bit-fields that holds the actions detected until now.
1043 * Attributes of flow that includes this action.
1045 * Pointer to error structure.
1048 * 0 on success, a negative errno value otherwise and rte_errno is set.
1051 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1052 const struct rte_flow_attr *attr,
1053 struct rte_flow_error *error)
1056 return rte_flow_error_set(error, ENOTSUP,
1057 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1058 "drop action not supported for "
1064 * Validate the queue action.
1067 * Pointer to the queue action.
1068 * @param[in] action_flags
1069 * Bit-fields that holds the actions detected until now.
1071 * Pointer to the Ethernet device structure.
1073 * Attributes of flow that includes this action.
1075 * Pointer to error structure.
1078 * 0 on success, a negative errno value otherwise and rte_errno is set.
1081 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1082 uint64_t action_flags,
1083 struct rte_eth_dev *dev,
1084 const struct rte_flow_attr *attr,
1085 struct rte_flow_error *error)
1087 struct mlx5_priv *priv = dev->data->dev_private;
1088 const struct rte_flow_action_queue *queue = action->conf;
1090 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1091 return rte_flow_error_set(error, EINVAL,
1092 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1093 "can't have 2 fate actions in"
1096 return rte_flow_error_set(error, EINVAL,
1097 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1098 NULL, "No Rx queues configured");
1099 if (queue->index >= priv->rxqs_n)
1100 return rte_flow_error_set(error, EINVAL,
1101 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1103 "queue index out of range");
1104 if (!(*priv->rxqs)[queue->index])
1105 return rte_flow_error_set(error, EINVAL,
1106 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1108 "queue is not configured");
1110 return rte_flow_error_set(error, ENOTSUP,
1111 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1112 "queue action not supported for "
1118 * Validate the rss action.
1121 * Pointer to the queue action.
1122 * @param[in] action_flags
1123 * Bit-fields that holds the actions detected until now.
1125 * Pointer to the Ethernet device structure.
1127 * Attributes of flow that includes this action.
1128 * @param[in] item_flags
1129 * Items that were detected.
1131 * Pointer to error structure.
1134 * 0 on success, a negative errno value otherwise and rte_errno is set.
1137 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1138 uint64_t action_flags,
1139 struct rte_eth_dev *dev,
1140 const struct rte_flow_attr *attr,
1141 uint64_t item_flags,
1142 struct rte_flow_error *error)
1144 struct mlx5_priv *priv = dev->data->dev_private;
1145 const struct rte_flow_action_rss *rss = action->conf;
1146 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1149 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1150 return rte_flow_error_set(error, EINVAL,
1151 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1152 "can't have 2 fate actions"
1154 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1155 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1156 return rte_flow_error_set(error, ENOTSUP,
1157 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1159 "RSS hash function not supported");
1160 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1165 return rte_flow_error_set(error, ENOTSUP,
1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1168 "tunnel RSS is not supported");
1169 /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1170 if (rss->key_len == 0 && rss->key != NULL)
1171 return rte_flow_error_set(error, ENOTSUP,
1172 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1174 "RSS hash key length 0");
1175 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1176 return rte_flow_error_set(error, ENOTSUP,
1177 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1179 "RSS hash key too small");
1180 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1181 return rte_flow_error_set(error, ENOTSUP,
1182 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1184 "RSS hash key too large");
1185 if (rss->queue_num > priv->config.ind_table_max_size)
1186 return rte_flow_error_set(error, ENOTSUP,
1187 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1189 "number of queues too large");
1190 if (rss->types & MLX5_RSS_HF_MASK)
1191 return rte_flow_error_set(error, ENOTSUP,
1192 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1194 "some RSS protocols are not"
1196 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1197 !(rss->types & ETH_RSS_IP))
1198 return rte_flow_error_set(error, EINVAL,
1199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1200 "L3 partial RSS requested but L3 RSS"
1201 " type not specified");
1202 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1203 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1204 return rte_flow_error_set(error, EINVAL,
1205 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1206 "L4 partial RSS requested but L4 RSS"
1207 " type not specified");
1209 return rte_flow_error_set(error, EINVAL,
1210 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1211 NULL, "No Rx queues configured");
1212 if (!rss->queue_num)
1213 return rte_flow_error_set(error, EINVAL,
1214 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1215 NULL, "No queues configured");
1216 for (i = 0; i != rss->queue_num; ++i) {
1217 if (rss->queue[i] >= priv->rxqs_n)
1218 return rte_flow_error_set
1220 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1221 &rss->queue[i], "queue index out of range");
1222 if (!(*priv->rxqs)[rss->queue[i]])
1223 return rte_flow_error_set
1224 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1225 &rss->queue[i], "queue is not configured");
1228 return rte_flow_error_set(error, ENOTSUP,
1229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1230 "rss action not supported for "
1232 if (rss->level > 1 && !tunnel)
1233 return rte_flow_error_set(error, EINVAL,
1234 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1235 "inner RSS is not supported for "
1236 "non-tunnel flows");
1241 * Validate the count action.
1244 * Pointer to the Ethernet device structure.
1246 * Attributes of flow that includes this action.
1248 * Pointer to error structure.
1251 * 0 on success, a negative errno value otherwise and rte_errno is set.
1254 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1255 const struct rte_flow_attr *attr,
1256 struct rte_flow_error *error)
1259 return rte_flow_error_set(error, ENOTSUP,
1260 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1261 "count action not supported for "
1267 * Verify the @p attributes will be correctly understood by the NIC and store
1268 * them in the @p flow if everything is correct.
1271 * Pointer to the Ethernet device structure.
1272 * @param[in] attributes
1273 * Pointer to flow attributes
1275 * Pointer to error structure.
1278 * 0 on success, a negative errno value otherwise and rte_errno is set.
1281 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1282 const struct rte_flow_attr *attributes,
1283 struct rte_flow_error *error)
1285 struct mlx5_priv *priv = dev->data->dev_private;
1286 uint32_t priority_max = priv->config.flow_prio - 1;
1288 if (attributes->group)
1289 return rte_flow_error_set(error, ENOTSUP,
1290 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1291 NULL, "groups is not supported");
1292 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
1293 attributes->priority >= priority_max)
1294 return rte_flow_error_set(error, ENOTSUP,
1295 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1296 NULL, "priority out of range");
1297 if (attributes->egress)
1298 return rte_flow_error_set(error, ENOTSUP,
1299 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1300 "egress is not supported");
1301 if (attributes->transfer && !priv->config.dv_esw_en)
1302 return rte_flow_error_set(error, ENOTSUP,
1303 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1304 NULL, "transfer is not supported");
1305 if (!attributes->ingress)
1306 return rte_flow_error_set(error, EINVAL,
1307 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1309 "ingress attribute is mandatory");
1314 * Validate ICMP6 item.
1317 * Item specification.
1318 * @param[in] item_flags
1319 * Bit-fields that holds the items detected until now.
1321 * Pointer to error structure.
1324 * 0 on success, a negative errno value otherwise and rte_errno is set.
1327 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1328 uint64_t item_flags,
1329 uint8_t target_protocol,
1330 struct rte_flow_error *error)
1332 const struct rte_flow_item_icmp6 *mask = item->mask;
1333 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1334 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1335 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1336 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1337 MLX5_FLOW_LAYER_OUTER_L4;
1340 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1341 return rte_flow_error_set(error, EINVAL,
1342 RTE_FLOW_ERROR_TYPE_ITEM, item,
1343 "protocol filtering not compatible"
1344 " with ICMP6 layer");
1345 if (!(item_flags & l3m))
1346 return rte_flow_error_set(error, EINVAL,
1347 RTE_FLOW_ERROR_TYPE_ITEM, item,
1348 "IPv6 is mandatory to filter on"
1350 if (item_flags & l4m)
1351 return rte_flow_error_set(error, EINVAL,
1352 RTE_FLOW_ERROR_TYPE_ITEM, item,
1353 "multiple L4 layers not supported");
1355 mask = &rte_flow_item_icmp6_mask;
1356 ret = mlx5_flow_item_acceptable
1357 (item, (const uint8_t *)mask,
1358 (const uint8_t *)&rte_flow_item_icmp6_mask,
1359 sizeof(struct rte_flow_item_icmp6), error);
1366 * Validate ICMP item.
1369 * Item specification.
1370 * @param[in] item_flags
1371 * Bit-fields that holds the items detected until now.
1373 * Pointer to error structure.
1376 * 0 on success, a negative errno value otherwise and rte_errno is set.
1379 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1380 uint64_t item_flags,
1381 uint8_t target_protocol,
1382 struct rte_flow_error *error)
1384 const struct rte_flow_item_icmp *mask = item->mask;
1385 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1386 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1387 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1388 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1389 MLX5_FLOW_LAYER_OUTER_L4;
1392 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1393 return rte_flow_error_set(error, EINVAL,
1394 RTE_FLOW_ERROR_TYPE_ITEM, item,
1395 "protocol filtering not compatible"
1396 " with ICMP layer");
1397 if (!(item_flags & l3m))
1398 return rte_flow_error_set(error, EINVAL,
1399 RTE_FLOW_ERROR_TYPE_ITEM, item,
1400 "IPv4 is mandatory to filter"
1402 if (item_flags & l4m)
1403 return rte_flow_error_set(error, EINVAL,
1404 RTE_FLOW_ERROR_TYPE_ITEM, item,
1405 "multiple L4 layers not supported");
1407 mask = &rte_flow_item_icmp_mask;
1408 ret = mlx5_flow_item_acceptable
1409 (item, (const uint8_t *)mask,
1410 (const uint8_t *)&rte_flow_item_icmp_mask,
1411 sizeof(struct rte_flow_item_icmp), error);
1418 * Validate Ethernet item.
1421 * Item specification.
1422 * @param[in] item_flags
1423 * Bit-fields that holds the items detected until now.
1425 * Pointer to error structure.
1428 * 0 on success, a negative errno value otherwise and rte_errno is set.
1431 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1432 uint64_t item_flags,
1433 struct rte_flow_error *error)
1435 const struct rte_flow_item_eth *mask = item->mask;
1436 const struct rte_flow_item_eth nic_mask = {
1437 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1438 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1439 .type = RTE_BE16(0xffff),
1442 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1443 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1444 MLX5_FLOW_LAYER_OUTER_L2;
1446 if (item_flags & ethm)
1447 return rte_flow_error_set(error, ENOTSUP,
1448 RTE_FLOW_ERROR_TYPE_ITEM, item,
1449 "multiple L2 layers not supported");
1450 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
1451 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
1452 return rte_flow_error_set(error, EINVAL,
1453 RTE_FLOW_ERROR_TYPE_ITEM, item,
1454 "L2 layer should not follow "
1456 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
1457 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
1458 return rte_flow_error_set(error, EINVAL,
1459 RTE_FLOW_ERROR_TYPE_ITEM, item,
1460 "L2 layer should not follow VLAN");
1462 mask = &rte_flow_item_eth_mask;
1463 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1464 (const uint8_t *)&nic_mask,
1465 sizeof(struct rte_flow_item_eth),
1471 * Validate VLAN item.
1474 * Item specification.
1475 * @param[in] item_flags
1476 * Bit-fields that holds the items detected until now.
1478 * Ethernet device flow is being created on.
1480 * Pointer to error structure.
1483 * 0 on success, a negative errno value otherwise and rte_errno is set.
1486 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
1487 uint64_t item_flags,
1488 struct rte_eth_dev *dev,
1489 struct rte_flow_error *error)
1491 const struct rte_flow_item_vlan *spec = item->spec;
1492 const struct rte_flow_item_vlan *mask = item->mask;
1493 const struct rte_flow_item_vlan nic_mask = {
1494 .tci = RTE_BE16(UINT16_MAX),
1495 .inner_type = RTE_BE16(UINT16_MAX),
1497 uint16_t vlan_tag = 0;
1498 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1500 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1501 MLX5_FLOW_LAYER_INNER_L4) :
1502 (MLX5_FLOW_LAYER_OUTER_L3 |
1503 MLX5_FLOW_LAYER_OUTER_L4);
1504 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1505 MLX5_FLOW_LAYER_OUTER_VLAN;
1507 if (item_flags & vlanm)
1508 return rte_flow_error_set(error, EINVAL,
1509 RTE_FLOW_ERROR_TYPE_ITEM, item,
1510 "multiple VLAN layers not supported");
1511 else if ((item_flags & l34m) != 0)
1512 return rte_flow_error_set(error, EINVAL,
1513 RTE_FLOW_ERROR_TYPE_ITEM, item,
1514 "VLAN cannot follow L3/L4 layer");
1516 mask = &rte_flow_item_vlan_mask;
1517 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1518 (const uint8_t *)&nic_mask,
1519 sizeof(struct rte_flow_item_vlan),
1523 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1524 struct mlx5_priv *priv = dev->data->dev_private;
1526 if (priv->vmwa_context) {
1528 * Non-NULL context means we have a virtual machine
1529 * and SR-IOV enabled, we have to create VLAN interface
1530 * to make hypervisor to setup E-Switch vport
1531 * context correctly. We avoid creating the multiple
1532 * VLAN interfaces, so we cannot support VLAN tag mask.
1534 return rte_flow_error_set(error, EINVAL,
1535 RTE_FLOW_ERROR_TYPE_ITEM,
1537 "VLAN tag mask is not"
1538 " supported in virtual"
1543 vlan_tag = spec->tci;
1544 vlan_tag &= mask->tci;
1547 * From verbs perspective an empty VLAN is equivalent
1548 * to a packet without VLAN layer.
1551 return rte_flow_error_set(error, EINVAL,
1552 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1554 "VLAN cannot be empty");
1559 * Validate IPV4 item.
1562 * Item specification.
1563 * @param[in] item_flags
1564 * Bit-fields that holds the items detected until now.
1565 * @param[in] acc_mask
1566 * Acceptable mask, if NULL default internal default mask
1567 * will be used to check whether item fields are supported.
1569 * Pointer to error structure.
1572 * 0 on success, a negative errno value otherwise and rte_errno is set.
1575 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
1576 uint64_t item_flags,
1578 uint16_t ether_type,
1579 const struct rte_flow_item_ipv4 *acc_mask,
1580 struct rte_flow_error *error)
1582 const struct rte_flow_item_ipv4 *mask = item->mask;
1583 const struct rte_flow_item_ipv4 *spec = item->spec;
1584 const struct rte_flow_item_ipv4 nic_mask = {
1586 .src_addr = RTE_BE32(0xffffffff),
1587 .dst_addr = RTE_BE32(0xffffffff),
1588 .type_of_service = 0xff,
1589 .next_proto_id = 0xff,
1592 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1593 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1594 MLX5_FLOW_LAYER_OUTER_L3;
1595 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1596 MLX5_FLOW_LAYER_OUTER_L4;
1598 uint8_t next_proto = 0xFF;
1599 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1600 MLX5_FLOW_LAYER_OUTER_VLAN |
1601 MLX5_FLOW_LAYER_INNER_VLAN);
1603 if ((last_item & l2_vlan) && ether_type &&
1604 ether_type != RTE_ETHER_TYPE_IPV4)
1605 return rte_flow_error_set(error, EINVAL,
1606 RTE_FLOW_ERROR_TYPE_ITEM, item,
1607 "IPv4 cannot follow L2/VLAN layer "
1608 "which ether type is not IPv4");
1609 if (item_flags & MLX5_FLOW_LAYER_IPIP) {
1611 next_proto = mask->hdr.next_proto_id &
1612 spec->hdr.next_proto_id;
1613 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1614 return rte_flow_error_set(error, EINVAL,
1615 RTE_FLOW_ERROR_TYPE_ITEM,
1620 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
1621 return rte_flow_error_set(error, EINVAL,
1622 RTE_FLOW_ERROR_TYPE_ITEM, item,
1623 "wrong tunnel type - IPv6 specified "
1624 "but IPv4 item provided");
1625 if (item_flags & l3m)
1626 return rte_flow_error_set(error, ENOTSUP,
1627 RTE_FLOW_ERROR_TYPE_ITEM, item,
1628 "multiple L3 layers not supported");
1629 else if (item_flags & l4m)
1630 return rte_flow_error_set(error, EINVAL,
1631 RTE_FLOW_ERROR_TYPE_ITEM, item,
1632 "L3 cannot follow an L4 layer.");
1633 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1634 !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1635 return rte_flow_error_set(error, EINVAL,
1636 RTE_FLOW_ERROR_TYPE_ITEM, item,
1637 "L3 cannot follow an NVGRE layer.");
1639 mask = &rte_flow_item_ipv4_mask;
1640 else if (mask->hdr.next_proto_id != 0 &&
1641 mask->hdr.next_proto_id != 0xff)
1642 return rte_flow_error_set(error, EINVAL,
1643 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
1644 "partial mask is not supported"
1646 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1647 acc_mask ? (const uint8_t *)acc_mask
1648 : (const uint8_t *)&nic_mask,
1649 sizeof(struct rte_flow_item_ipv4),
1657 * Validate IPV6 item.
1660 * Item specification.
1661 * @param[in] item_flags
1662 * Bit-fields that holds the items detected until now.
1663 * @param[in] acc_mask
1664 * Acceptable mask, if NULL default internal default mask
1665 * will be used to check whether item fields are supported.
1667 * Pointer to error structure.
1670 * 0 on success, a negative errno value otherwise and rte_errno is set.
1673 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
1674 uint64_t item_flags,
1676 uint16_t ether_type,
1677 const struct rte_flow_item_ipv6 *acc_mask,
1678 struct rte_flow_error *error)
1680 const struct rte_flow_item_ipv6 *mask = item->mask;
1681 const struct rte_flow_item_ipv6 *spec = item->spec;
1682 const struct rte_flow_item_ipv6 nic_mask = {
1685 "\xff\xff\xff\xff\xff\xff\xff\xff"
1686 "\xff\xff\xff\xff\xff\xff\xff\xff",
1688 "\xff\xff\xff\xff\xff\xff\xff\xff"
1689 "\xff\xff\xff\xff\xff\xff\xff\xff",
1690 .vtc_flow = RTE_BE32(0xffffffff),
1694 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1695 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1696 MLX5_FLOW_LAYER_OUTER_L3;
1697 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1698 MLX5_FLOW_LAYER_OUTER_L4;
1700 uint8_t next_proto = 0xFF;
1701 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
1702 MLX5_FLOW_LAYER_OUTER_VLAN |
1703 MLX5_FLOW_LAYER_INNER_VLAN);
1705 if ((last_item & l2_vlan) && ether_type &&
1706 ether_type != RTE_ETHER_TYPE_IPV6)
1707 return rte_flow_error_set(error, EINVAL,
1708 RTE_FLOW_ERROR_TYPE_ITEM, item,
1709 "IPv6 cannot follow L2/VLAN layer "
1710 "which ether type is not IPv6");
1711 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) {
1713 next_proto = mask->hdr.proto & spec->hdr.proto;
1714 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
1715 return rte_flow_error_set(error, EINVAL,
1716 RTE_FLOW_ERROR_TYPE_ITEM,
1721 if (item_flags & MLX5_FLOW_LAYER_IPIP)
1722 return rte_flow_error_set(error, EINVAL,
1723 RTE_FLOW_ERROR_TYPE_ITEM, item,
1724 "wrong tunnel type - IPv4 specified "
1725 "but IPv6 item provided");
1726 if (item_flags & l3m)
1727 return rte_flow_error_set(error, ENOTSUP,
1728 RTE_FLOW_ERROR_TYPE_ITEM, item,
1729 "multiple L3 layers not supported");
1730 else if (item_flags & l4m)
1731 return rte_flow_error_set(error, EINVAL,
1732 RTE_FLOW_ERROR_TYPE_ITEM, item,
1733 "L3 cannot follow an L4 layer.");
1734 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
1735 !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
1736 return rte_flow_error_set(error, EINVAL,
1737 RTE_FLOW_ERROR_TYPE_ITEM, item,
1738 "L3 cannot follow an NVGRE layer.");
1740 mask = &rte_flow_item_ipv6_mask;
1741 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1742 acc_mask ? (const uint8_t *)acc_mask
1743 : (const uint8_t *)&nic_mask,
1744 sizeof(struct rte_flow_item_ipv6),
1752 * Validate UDP item.
1755 * Item specification.
1756 * @param[in] item_flags
1757 * Bit-fields that holds the items detected until now.
1758 * @param[in] target_protocol
1759 * The next protocol in the previous item.
1760 * @param[in] flow_mask
1761 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
1763 * Pointer to error structure.
1766 * 0 on success, a negative errno value otherwise and rte_errno is set.
1769 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
1770 uint64_t item_flags,
1771 uint8_t target_protocol,
1772 struct rte_flow_error *error)
1774 const struct rte_flow_item_udp *mask = item->mask;
1775 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1776 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1777 MLX5_FLOW_LAYER_OUTER_L3;
1778 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1779 MLX5_FLOW_LAYER_OUTER_L4;
1782 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
1783 return rte_flow_error_set(error, EINVAL,
1784 RTE_FLOW_ERROR_TYPE_ITEM, item,
1785 "protocol filtering not compatible"
1787 if (!(item_flags & l3m))
1788 return rte_flow_error_set(error, EINVAL,
1789 RTE_FLOW_ERROR_TYPE_ITEM, item,
1790 "L3 is mandatory to filter on L4");
1791 if (item_flags & l4m)
1792 return rte_flow_error_set(error, EINVAL,
1793 RTE_FLOW_ERROR_TYPE_ITEM, item,
1794 "multiple L4 layers not supported");
1796 mask = &rte_flow_item_udp_mask;
1797 ret = mlx5_flow_item_acceptable
1798 (item, (const uint8_t *)mask,
1799 (const uint8_t *)&rte_flow_item_udp_mask,
1800 sizeof(struct rte_flow_item_udp), error);
1807 * Validate TCP item.
1810 * Item specification.
1811 * @param[in] item_flags
1812 * Bit-fields that holds the items detected until now.
1813 * @param[in] target_protocol
1814 * The next protocol in the previous item.
1816 * Pointer to error structure.
1819 * 0 on success, a negative errno value otherwise and rte_errno is set.
1822 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
1823 uint64_t item_flags,
1824 uint8_t target_protocol,
1825 const struct rte_flow_item_tcp *flow_mask,
1826 struct rte_flow_error *error)
1828 const struct rte_flow_item_tcp *mask = item->mask;
1829 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1830 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
1831 MLX5_FLOW_LAYER_OUTER_L3;
1832 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1833 MLX5_FLOW_LAYER_OUTER_L4;
1836 MLX5_ASSERT(flow_mask);
1837 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
1838 return rte_flow_error_set(error, EINVAL,
1839 RTE_FLOW_ERROR_TYPE_ITEM, item,
1840 "protocol filtering not compatible"
1842 if (!(item_flags & l3m))
1843 return rte_flow_error_set(error, EINVAL,
1844 RTE_FLOW_ERROR_TYPE_ITEM, item,
1845 "L3 is mandatory to filter on L4");
1846 if (item_flags & l4m)
1847 return rte_flow_error_set(error, EINVAL,
1848 RTE_FLOW_ERROR_TYPE_ITEM, item,
1849 "multiple L4 layers not supported");
1851 mask = &rte_flow_item_tcp_mask;
1852 ret = mlx5_flow_item_acceptable
1853 (item, (const uint8_t *)mask,
1854 (const uint8_t *)flow_mask,
1855 sizeof(struct rte_flow_item_tcp), error);
1862 * Validate VXLAN item.
1865 * Item specification.
1866 * @param[in] item_flags
1867 * Bit-fields that holds the items detected until now.
1868 * @param[in] target_protocol
1869 * The next protocol in the previous item.
1871 * Pointer to error structure.
1874 * 0 on success, a negative errno value otherwise and rte_errno is set.
1877 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
1878 uint64_t item_flags,
1879 struct rte_flow_error *error)
1881 const struct rte_flow_item_vxlan *spec = item->spec;
1882 const struct rte_flow_item_vxlan *mask = item->mask;
1887 } id = { .vlan_id = 0, };
1890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1891 return rte_flow_error_set(error, ENOTSUP,
1892 RTE_FLOW_ERROR_TYPE_ITEM, item,
1893 "multiple tunnel layers not"
1896 * Verify only UDPv4 is present as defined in
1897 * https://tools.ietf.org/html/rfc7348
1899 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1900 return rte_flow_error_set(error, EINVAL,
1901 RTE_FLOW_ERROR_TYPE_ITEM, item,
1902 "no outer UDP layer found");
1904 mask = &rte_flow_item_vxlan_mask;
1905 ret = mlx5_flow_item_acceptable
1906 (item, (const uint8_t *)mask,
1907 (const uint8_t *)&rte_flow_item_vxlan_mask,
1908 sizeof(struct rte_flow_item_vxlan),
1913 memcpy(&id.vni[1], spec->vni, 3);
1914 memcpy(&id.vni[1], mask->vni, 3);
1916 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1917 return rte_flow_error_set(error, ENOTSUP,
1918 RTE_FLOW_ERROR_TYPE_ITEM, item,
1919 "VXLAN tunnel must be fully defined");
1924 * Validate VXLAN_GPE item.
1927 * Item specification.
1928 * @param[in] item_flags
1929 * Bit-fields that holds the items detected until now.
1931 * Pointer to the private data structure.
1932 * @param[in] target_protocol
1933 * The next protocol in the previous item.
1935 * Pointer to error structure.
1938 * 0 on success, a negative errno value otherwise and rte_errno is set.
1941 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
1942 uint64_t item_flags,
1943 struct rte_eth_dev *dev,
1944 struct rte_flow_error *error)
1946 struct mlx5_priv *priv = dev->data->dev_private;
1947 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
1948 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
1953 } id = { .vlan_id = 0, };
1955 if (!priv->config.l3_vxlan_en)
1956 return rte_flow_error_set(error, ENOTSUP,
1957 RTE_FLOW_ERROR_TYPE_ITEM, item,
1958 "L3 VXLAN is not enabled by device"
1959 " parameter and/or not configured in"
1961 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1962 return rte_flow_error_set(error, ENOTSUP,
1963 RTE_FLOW_ERROR_TYPE_ITEM, item,
1964 "multiple tunnel layers not"
1967 * Verify only UDPv4 is present as defined in
1968 * https://tools.ietf.org/html/rfc7348
1970 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1971 return rte_flow_error_set(error, EINVAL,
1972 RTE_FLOW_ERROR_TYPE_ITEM, item,
1973 "no outer UDP layer found");
1975 mask = &rte_flow_item_vxlan_gpe_mask;
1976 ret = mlx5_flow_item_acceptable
1977 (item, (const uint8_t *)mask,
1978 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
1979 sizeof(struct rte_flow_item_vxlan_gpe),
1985 return rte_flow_error_set(error, ENOTSUP,
1986 RTE_FLOW_ERROR_TYPE_ITEM,
1988 "VxLAN-GPE protocol"
1990 memcpy(&id.vni[1], spec->vni, 3);
1991 memcpy(&id.vni[1], mask->vni, 3);
1993 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
1994 return rte_flow_error_set(error, ENOTSUP,
1995 RTE_FLOW_ERROR_TYPE_ITEM, item,
1996 "VXLAN-GPE tunnel must be fully"
2001 * Validate GRE Key item.
2004 * Item specification.
2005 * @param[in] item_flags
2006 * Bit flags to mark detected items.
2007 * @param[in] gre_item
2008 * Pointer to gre_item
2010 * Pointer to error structure.
2013 * 0 on success, a negative errno value otherwise and rte_errno is set.
2016 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2017 uint64_t item_flags,
2018 const struct rte_flow_item *gre_item,
2019 struct rte_flow_error *error)
2021 const rte_be32_t *mask = item->mask;
2023 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2024 const struct rte_flow_item_gre *gre_spec;
2025 const struct rte_flow_item_gre *gre_mask;
2027 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2028 return rte_flow_error_set(error, ENOTSUP,
2029 RTE_FLOW_ERROR_TYPE_ITEM, item,
2030 "Multiple GRE key not support");
2031 if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2032 return rte_flow_error_set(error, ENOTSUP,
2033 RTE_FLOW_ERROR_TYPE_ITEM, item,
2034 "No preceding GRE header");
2035 if (item_flags & MLX5_FLOW_LAYER_INNER)
2036 return rte_flow_error_set(error, ENOTSUP,
2037 RTE_FLOW_ERROR_TYPE_ITEM, item,
2038 "GRE key following a wrong item");
2039 gre_mask = gre_item->mask;
2041 gre_mask = &rte_flow_item_gre_mask;
2042 gre_spec = gre_item->spec;
2043 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2044 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2045 return rte_flow_error_set(error, EINVAL,
2046 RTE_FLOW_ERROR_TYPE_ITEM, item,
2047 "Key bit must be on");
2050 mask = &gre_key_default_mask;
2051 ret = mlx5_flow_item_acceptable
2052 (item, (const uint8_t *)mask,
2053 (const uint8_t *)&gre_key_default_mask,
2054 sizeof(rte_be32_t), error);
2059 * Validate GRE item.
2062 * Item specification.
2063 * @param[in] item_flags
2064 * Bit flags to mark detected items.
2065 * @param[in] target_protocol
2066 * The next protocol in the previous item.
2068 * Pointer to error structure.
2071 * 0 on success, a negative errno value otherwise and rte_errno is set.
2074 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2075 uint64_t item_flags,
2076 uint8_t target_protocol,
2077 struct rte_flow_error *error)
2079 const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2080 const struct rte_flow_item_gre *mask = item->mask;
2082 const struct rte_flow_item_gre nic_mask = {
2083 .c_rsvd0_ver = RTE_BE16(0xB000),
2084 .protocol = RTE_BE16(UINT16_MAX),
2087 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2088 return rte_flow_error_set(error, EINVAL,
2089 RTE_FLOW_ERROR_TYPE_ITEM, item,
2090 "protocol filtering not compatible"
2091 " with this GRE layer");
2092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2093 return rte_flow_error_set(error, ENOTSUP,
2094 RTE_FLOW_ERROR_TYPE_ITEM, item,
2095 "multiple tunnel layers not"
2097 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2098 return rte_flow_error_set(error, ENOTSUP,
2099 RTE_FLOW_ERROR_TYPE_ITEM, item,
2100 "L3 Layer is missing");
2102 mask = &rte_flow_item_gre_mask;
2103 ret = mlx5_flow_item_acceptable
2104 (item, (const uint8_t *)mask,
2105 (const uint8_t *)&nic_mask,
2106 sizeof(struct rte_flow_item_gre), error);
2109 #ifndef HAVE_MLX5DV_DR
2110 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2111 if (spec && (spec->protocol & mask->protocol))
2112 return rte_flow_error_set(error, ENOTSUP,
2113 RTE_FLOW_ERROR_TYPE_ITEM, item,
2114 "without MPLS support the"
2115 " specification cannot be used for"
2123 * Validate Geneve item.
2126 * Item specification.
2127 * @param[in] itemFlags
2128 * Bit-fields that holds the items detected until now.
2130 * Pointer to the private data structure.
2132 * Pointer to error structure.
2135 * 0 on success, a negative errno value otherwise and rte_errno is set.
2139 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2140 uint64_t item_flags,
2141 struct rte_eth_dev *dev,
2142 struct rte_flow_error *error)
2144 struct mlx5_priv *priv = dev->data->dev_private;
2145 const struct rte_flow_item_geneve *spec = item->spec;
2146 const struct rte_flow_item_geneve *mask = item->mask;
2149 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2150 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2151 const struct rte_flow_item_geneve nic_mask = {
2152 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2153 .vni = "\xff\xff\xff",
2154 .protocol = RTE_BE16(UINT16_MAX),
2157 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2158 return rte_flow_error_set(error, ENOTSUP,
2159 RTE_FLOW_ERROR_TYPE_ITEM, item,
2160 "L3 Geneve is not enabled by device"
2161 " parameter and/or not configured in"
2163 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2164 return rte_flow_error_set(error, ENOTSUP,
2165 RTE_FLOW_ERROR_TYPE_ITEM, item,
2166 "multiple tunnel layers not"
2169 * Verify only UDPv4 is present as defined in
2170 * https://tools.ietf.org/html/rfc7348
2172 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2173 return rte_flow_error_set(error, EINVAL,
2174 RTE_FLOW_ERROR_TYPE_ITEM, item,
2175 "no outer UDP layer found");
2177 mask = &rte_flow_item_geneve_mask;
2178 ret = mlx5_flow_item_acceptable
2179 (item, (const uint8_t *)mask,
2180 (const uint8_t *)&nic_mask,
2181 sizeof(struct rte_flow_item_geneve), error);
2185 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2186 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2187 MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2188 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2189 return rte_flow_error_set(error, ENOTSUP,
2190 RTE_FLOW_ERROR_TYPE_ITEM,
2192 "Geneve protocol unsupported"
2193 " fields are being used");
2194 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2195 return rte_flow_error_set
2197 RTE_FLOW_ERROR_TYPE_ITEM,
2199 "Unsupported Geneve options length");
2201 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2202 return rte_flow_error_set
2204 RTE_FLOW_ERROR_TYPE_ITEM, item,
2205 "Geneve tunnel must be fully defined");
2210 * Validate MPLS item.
2213 * Pointer to the rte_eth_dev structure.
2215 * Item specification.
2216 * @param[in] item_flags
2217 * Bit-fields that holds the items detected until now.
2218 * @param[in] prev_layer
2219 * The protocol layer indicated in previous item.
2221 * Pointer to error structure.
2224 * 0 on success, a negative errno value otherwise and rte_errno is set.
2227 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2228 const struct rte_flow_item *item __rte_unused,
2229 uint64_t item_flags __rte_unused,
2230 uint64_t prev_layer __rte_unused,
2231 struct rte_flow_error *error)
2233 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2234 const struct rte_flow_item_mpls *mask = item->mask;
2235 struct mlx5_priv *priv = dev->data->dev_private;
2238 if (!priv->config.mpls_en)
2239 return rte_flow_error_set(error, ENOTSUP,
2240 RTE_FLOW_ERROR_TYPE_ITEM, item,
2241 "MPLS not supported or"
2242 " disabled in firmware"
2244 /* MPLS over IP, UDP, GRE is allowed */
2245 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 |
2246 MLX5_FLOW_LAYER_OUTER_L4_UDP |
2247 MLX5_FLOW_LAYER_GRE)))
2248 return rte_flow_error_set(error, EINVAL,
2249 RTE_FLOW_ERROR_TYPE_ITEM, item,
2250 "protocol filtering not compatible"
2251 " with MPLS layer");
2252 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
2253 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
2254 !(item_flags & MLX5_FLOW_LAYER_GRE))
2255 return rte_flow_error_set(error, ENOTSUP,
2256 RTE_FLOW_ERROR_TYPE_ITEM, item,
2257 "multiple tunnel layers not"
2260 mask = &rte_flow_item_mpls_mask;
2261 ret = mlx5_flow_item_acceptable
2262 (item, (const uint8_t *)mask,
2263 (const uint8_t *)&rte_flow_item_mpls_mask,
2264 sizeof(struct rte_flow_item_mpls), error);
2269 return rte_flow_error_set(error, ENOTSUP,
2270 RTE_FLOW_ERROR_TYPE_ITEM, item,
2271 "MPLS is not supported by Verbs, please"
2276 * Validate NVGRE item.
2279 * Item specification.
2280 * @param[in] item_flags
2281 * Bit flags to mark detected items.
2282 * @param[in] target_protocol
2283 * The next protocol in the previous item.
2285 * Pointer to error structure.
2288 * 0 on success, a negative errno value otherwise and rte_errno is set.
2291 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
2292 uint64_t item_flags,
2293 uint8_t target_protocol,
2294 struct rte_flow_error *error)
2296 const struct rte_flow_item_nvgre *mask = item->mask;
2299 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2300 return rte_flow_error_set(error, EINVAL,
2301 RTE_FLOW_ERROR_TYPE_ITEM, item,
2302 "protocol filtering not compatible"
2303 " with this GRE layer");
2304 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2305 return rte_flow_error_set(error, ENOTSUP,
2306 RTE_FLOW_ERROR_TYPE_ITEM, item,
2307 "multiple tunnel layers not"
2309 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2310 return rte_flow_error_set(error, ENOTSUP,
2311 RTE_FLOW_ERROR_TYPE_ITEM, item,
2312 "L3 Layer is missing");
2314 mask = &rte_flow_item_nvgre_mask;
2315 ret = mlx5_flow_item_acceptable
2316 (item, (const uint8_t *)mask,
2317 (const uint8_t *)&rte_flow_item_nvgre_mask,
2318 sizeof(struct rte_flow_item_nvgre), error);
2324 /* Allocate unique ID for the split Q/RSS subflows. */
2326 flow_qrss_get_id(struct rte_eth_dev *dev)
2328 struct mlx5_priv *priv = dev->data->dev_private;
2329 uint32_t qrss_id, ret;
2331 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id);
2334 MLX5_ASSERT(qrss_id);
2338 /* Free unique ID for the split Q/RSS subflows. */
2340 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id)
2342 struct mlx5_priv *priv = dev->data->dev_private;
2345 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id);
2349 * Release resource related QUEUE/RSS action split.
2352 * Pointer to Ethernet device.
2354 * Flow to release id's from.
2357 flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
2358 struct rte_flow *flow)
2360 struct mlx5_priv *priv = dev->data->dev_private;
2361 uint32_t handle_idx;
2362 struct mlx5_flow_handle *dev_handle;
2364 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2365 handle_idx, dev_handle, next)
2366 if (dev_handle->split_flow_id)
2367 flow_qrss_free_id(dev, dev_handle->split_flow_id);
2371 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
2372 const struct rte_flow_attr *attr __rte_unused,
2373 const struct rte_flow_item items[] __rte_unused,
2374 const struct rte_flow_action actions[] __rte_unused,
2375 bool external __rte_unused,
2376 int hairpin __rte_unused,
2377 struct rte_flow_error *error)
2379 return rte_flow_error_set(error, ENOTSUP,
2380 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2383 static struct mlx5_flow *
2384 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
2385 const struct rte_flow_attr *attr __rte_unused,
2386 const struct rte_flow_item items[] __rte_unused,
2387 const struct rte_flow_action actions[] __rte_unused,
2388 struct rte_flow_error *error)
2390 rte_flow_error_set(error, ENOTSUP,
2391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2396 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
2397 struct mlx5_flow *dev_flow __rte_unused,
2398 const struct rte_flow_attr *attr __rte_unused,
2399 const struct rte_flow_item items[] __rte_unused,
2400 const struct rte_flow_action actions[] __rte_unused,
2401 struct rte_flow_error *error)
2403 return rte_flow_error_set(error, ENOTSUP,
2404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2408 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
2409 struct rte_flow *flow __rte_unused,
2410 struct rte_flow_error *error)
2412 return rte_flow_error_set(error, ENOTSUP,
2413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2417 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
2418 struct rte_flow *flow __rte_unused)
2423 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
2424 struct rte_flow *flow __rte_unused)
2429 flow_null_query(struct rte_eth_dev *dev __rte_unused,
2430 struct rte_flow *flow __rte_unused,
2431 const struct rte_flow_action *actions __rte_unused,
2432 void *data __rte_unused,
2433 struct rte_flow_error *error)
2435 return rte_flow_error_set(error, ENOTSUP,
2436 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
2439 /* Void driver to protect from null pointer reference. */
2440 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
2441 .validate = flow_null_validate,
2442 .prepare = flow_null_prepare,
2443 .translate = flow_null_translate,
2444 .apply = flow_null_apply,
2445 .remove = flow_null_remove,
2446 .destroy = flow_null_destroy,
2447 .query = flow_null_query,
2451 * Select flow driver type according to flow attributes and device
2455 * Pointer to the dev structure.
2457 * Pointer to the flow attributes.
2460 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
2462 static enum mlx5_flow_drv_type
2463 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
2465 struct mlx5_priv *priv = dev->data->dev_private;
2466 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX;
2468 if (attr->transfer && priv->config.dv_esw_en)
2469 type = MLX5_FLOW_TYPE_DV;
2470 if (!attr->transfer)
2471 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
2472 MLX5_FLOW_TYPE_VERBS;
2476 #define flow_get_drv_ops(type) flow_drv_ops[type]
2479 * Flow driver validation API. This abstracts calling driver specific functions.
2480 * The type of flow driver is determined according to flow attributes.
2483 * Pointer to the dev structure.
2485 * Pointer to the flow attributes.
2487 * Pointer to the list of items.
2488 * @param[in] actions
2489 * Pointer to the list of actions.
2490 * @param[in] external
2491 * This flow rule is created by request external to PMD.
2492 * @param[in] hairpin
2493 * Number of hairpin TX actions, 0 means classic flow.
2495 * Pointer to the error structure.
2498 * 0 on success, a negative errno value otherwise and rte_errno is set.
2501 flow_drv_validate(struct rte_eth_dev *dev,
2502 const struct rte_flow_attr *attr,
2503 const struct rte_flow_item items[],
2504 const struct rte_flow_action actions[],
2505 bool external, int hairpin, struct rte_flow_error *error)
2507 const struct mlx5_flow_driver_ops *fops;
2508 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
2510 fops = flow_get_drv_ops(type);
2511 return fops->validate(dev, attr, items, actions, external,
2516 * Flow driver preparation API. This abstracts calling driver specific
2517 * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2518 * calculates the size of memory required for device flow, allocates the memory,
2519 * initializes the device flow and returns the pointer.
2522 * This function initializes device flow structure such as dv or verbs in
2523 * struct mlx5_flow. However, it is caller's responsibility to initialize the
2524 * rest. For example, adding returning device flow to flow->dev_flow list and
2525 * setting backward reference to the flow should be done out of this function.
2526 * layers field is not filled either.
2529 * Pointer to the dev structure.
2531 * Pointer to the flow attributes.
2533 * Pointer to the list of items.
2534 * @param[in] actions
2535 * Pointer to the list of actions.
2536 * @param[in] flow_idx
2537 * This memory pool index to the flow.
2539 * Pointer to the error structure.
2542 * Pointer to device flow on success, otherwise NULL and rte_errno is set.
2544 static inline struct mlx5_flow *
2545 flow_drv_prepare(struct rte_eth_dev *dev,
2546 const struct rte_flow *flow,
2547 const struct rte_flow_attr *attr,
2548 const struct rte_flow_item items[],
2549 const struct rte_flow_action actions[],
2551 struct rte_flow_error *error)
2553 const struct mlx5_flow_driver_ops *fops;
2554 enum mlx5_flow_drv_type type = flow->drv_type;
2555 struct mlx5_flow *mlx5_flow = NULL;
2557 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2558 fops = flow_get_drv_ops(type);
2559 mlx5_flow = fops->prepare(dev, attr, items, actions, error);
2561 mlx5_flow->flow_idx = flow_idx;
2566 * Flow driver translation API. This abstracts calling driver specific
2567 * functions. Parent flow (rte_flow) should have driver type (drv_type). It
2568 * translates a generic flow into a driver flow. flow_drv_prepare() must
2572 * dev_flow->layers could be filled as a result of parsing during translation
2573 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
2574 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
2575 * flow->actions could be overwritten even though all the expanded dev_flows
2576 * have the same actions.
2579 * Pointer to the rte dev structure.
2580 * @param[in, out] dev_flow
2581 * Pointer to the mlx5 flow.
2583 * Pointer to the flow attributes.
2585 * Pointer to the list of items.
2586 * @param[in] actions
2587 * Pointer to the list of actions.
2589 * Pointer to the error structure.
2592 * 0 on success, a negative errno value otherwise and rte_errno is set.
2595 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
2596 const struct rte_flow_attr *attr,
2597 const struct rte_flow_item items[],
2598 const struct rte_flow_action actions[],
2599 struct rte_flow_error *error)
2601 const struct mlx5_flow_driver_ops *fops;
2602 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
2604 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2605 fops = flow_get_drv_ops(type);
2606 return fops->translate(dev, dev_flow, attr, items, actions, error);
2610 * Flow driver apply API. This abstracts calling driver specific functions.
2611 * Parent flow (rte_flow) should have driver type (drv_type). It applies
2612 * translated driver flows on to device. flow_drv_translate() must precede.
2615 * Pointer to Ethernet device structure.
2616 * @param[in, out] flow
2617 * Pointer to flow structure.
2619 * Pointer to error structure.
2622 * 0 on success, a negative errno value otherwise and rte_errno is set.
2625 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
2626 struct rte_flow_error *error)
2628 const struct mlx5_flow_driver_ops *fops;
2629 enum mlx5_flow_drv_type type = flow->drv_type;
2631 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2632 fops = flow_get_drv_ops(type);
2633 return fops->apply(dev, flow, error);
2637 * Flow driver remove API. This abstracts calling driver specific functions.
2638 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2639 * on device. All the resources of the flow should be freed by calling
2640 * flow_drv_destroy().
2643 * Pointer to Ethernet device.
2644 * @param[in, out] flow
2645 * Pointer to flow structure.
2648 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
2650 const struct mlx5_flow_driver_ops *fops;
2651 enum mlx5_flow_drv_type type = flow->drv_type;
2653 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2654 fops = flow_get_drv_ops(type);
2655 fops->remove(dev, flow);
2659 * Flow driver destroy API. This abstracts calling driver specific functions.
2660 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
2661 * on device and releases resources of the flow.
2664 * Pointer to Ethernet device.
2665 * @param[in, out] flow
2666 * Pointer to flow structure.
2669 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
2671 const struct mlx5_flow_driver_ops *fops;
2672 enum mlx5_flow_drv_type type = flow->drv_type;
2674 flow_mreg_split_qrss_release(dev, flow);
2675 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
2676 fops = flow_get_drv_ops(type);
2677 fops->destroy(dev, flow);
2681 * Get RSS action from the action list.
2683 * @param[in] actions
2684 * Pointer to the list of actions.
2687 * Pointer to the RSS action if exist, else return NULL.
2689 static const struct rte_flow_action_rss*
2690 flow_get_rss_action(const struct rte_flow_action actions[])
2692 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2693 switch (actions->type) {
2694 case RTE_FLOW_ACTION_TYPE_RSS:
2695 return (const struct rte_flow_action_rss *)
2705 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
2707 const struct rte_flow_item *item;
2708 unsigned int has_vlan = 0;
2710 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
2711 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
2717 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
2718 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
2719 return rss_level < 2 ? MLX5_EXPANSION_ROOT :
2720 MLX5_EXPANSION_ROOT_OUTER;
2724 * Get layer flags from the prefix flow.
2726 * Some flows may be split to several subflows, the prefix subflow gets the
2727 * match items and the suffix sub flow gets the actions.
2728 * Some actions need the user defined match item flags to get the detail for
2730 * This function helps the suffix flow to get the item layer flags from prefix
2733 * @param[in] dev_flow
2734 * Pointer the created preifx subflow.
2737 * The layers get from prefix subflow.
2739 static inline uint64_t
2740 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
2742 uint64_t layers = 0;
2745 * Layers bits could be localization, but usually the compiler will
2746 * help to do the optimization work for source code.
2747 * If no decap actions, use the layers directly.
2749 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
2750 return dev_flow->handle->layers;
2751 /* Convert L3 layers with decap action. */
2752 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
2753 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2754 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
2755 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2756 /* Convert L4 layers with decap action. */
2757 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
2758 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
2759 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
2760 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
2765 * Get metadata split action information.
2767 * @param[in] actions
2768 * Pointer to the list of actions.
2770 * Pointer to the return pointer.
2771 * @param[out] qrss_type
2772 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
2773 * if no QUEUE/RSS is found.
2774 * @param[out] encap_idx
2775 * Pointer to the index of the encap action if exists, otherwise the last
2779 * Total number of actions.
2782 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
2783 const struct rte_flow_action **qrss,
2786 const struct rte_flow_action_raw_encap *raw_encap;
2788 int raw_decap_idx = -1;
2791 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2792 switch (actions->type) {
2793 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2794 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2795 *encap_idx = actions_n;
2797 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
2798 raw_decap_idx = actions_n;
2800 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2801 raw_encap = actions->conf;
2802 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2803 *encap_idx = raw_decap_idx != -1 ?
2804 raw_decap_idx : actions_n;
2806 case RTE_FLOW_ACTION_TYPE_QUEUE:
2807 case RTE_FLOW_ACTION_TYPE_RSS:
2815 if (*encap_idx == -1)
2816 *encap_idx = actions_n;
2817 /* Count RTE_FLOW_ACTION_TYPE_END. */
2818 return actions_n + 1;
2822 * Check meter action from the action list.
2824 * @param[in] actions
2825 * Pointer to the list of actions.
2827 * Pointer to the meter exist flag.
2830 * Total number of actions.
2833 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
2839 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2840 switch (actions->type) {
2841 case RTE_FLOW_ACTION_TYPE_METER:
2849 /* Count RTE_FLOW_ACTION_TYPE_END. */
2850 return actions_n + 1;
2854 * Check if the flow should be splited due to hairpin.
2855 * The reason for the split is that in current HW we can't
2856 * support encap on Rx, so if a flow have encap we move it
2860 * Pointer to Ethernet device.
2862 * Flow rule attributes.
2863 * @param[in] actions
2864 * Associated actions (list terminated by the END action).
2867 * > 0 the number of actions and the flow should be split,
2868 * 0 when no split required.
2871 flow_check_hairpin_split(struct rte_eth_dev *dev,
2872 const struct rte_flow_attr *attr,
2873 const struct rte_flow_action actions[])
2875 int queue_action = 0;
2878 const struct rte_flow_action_queue *queue;
2879 const struct rte_flow_action_rss *rss;
2880 const struct rte_flow_action_raw_encap *raw_encap;
2884 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2885 switch (actions->type) {
2886 case RTE_FLOW_ACTION_TYPE_QUEUE:
2887 queue = actions->conf;
2890 if (mlx5_rxq_get_type(dev, queue->index) !=
2891 MLX5_RXQ_TYPE_HAIRPIN)
2896 case RTE_FLOW_ACTION_TYPE_RSS:
2897 rss = actions->conf;
2898 if (rss == NULL || rss->queue_num == 0)
2900 if (mlx5_rxq_get_type(dev, rss->queue[0]) !=
2901 MLX5_RXQ_TYPE_HAIRPIN)
2906 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2907 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2911 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2912 raw_encap = actions->conf;
2913 if (raw_encap->size >
2914 (sizeof(struct rte_flow_item_eth) +
2915 sizeof(struct rte_flow_item_ipv4)))
2924 if (encap == 1 && queue_action)
2929 /* Declare flow create/destroy prototype in advance. */
2931 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
2932 const struct rte_flow_attr *attr,
2933 const struct rte_flow_item items[],
2934 const struct rte_flow_action actions[],
2935 bool external, struct rte_flow_error *error);
2938 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
2942 * Add a flow of copying flow metadata registers in RX_CP_TBL.
2944 * As mark_id is unique, if there's already a registered flow for the mark_id,
2945 * return by increasing the reference counter of the resource. Otherwise, create
2946 * the resource (mcp_res) and flow.
2949 * - If ingress port is ANY and reg_c[1] is mark_id,
2950 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
2952 * For default flow (zero mark_id), flow is like,
2953 * - If ingress port is ANY,
2954 * reg_b := reg_c[0] and jump to RX_ACT_TBL.
2957 * Pointer to Ethernet device.
2959 * ID of MARK action, zero means default flow for META.
2961 * Perform verbose error reporting if not NULL.
2964 * Associated resource on success, NULL otherwise and rte_errno is set.
2966 static struct mlx5_flow_mreg_copy_resource *
2967 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
2968 struct rte_flow_error *error)
2970 struct mlx5_priv *priv = dev->data->dev_private;
2971 struct rte_flow_attr attr = {
2972 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
2975 struct mlx5_rte_flow_item_tag tag_spec = {
2978 struct rte_flow_item items[] = {
2979 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
2981 struct rte_flow_action_mark ftag = {
2984 struct mlx5_flow_action_copy_mreg cp_mreg = {
2988 struct rte_flow_action_jump jump = {
2989 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
2991 struct rte_flow_action actions[] = {
2992 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
2994 struct mlx5_flow_mreg_copy_resource *mcp_res;
2998 /* Fill the register fileds in the flow. */
2999 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3003 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3007 /* Check if already registered. */
3008 MLX5_ASSERT(priv->mreg_cp_tbl);
3009 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
3011 /* For non-default rule. */
3012 if (mark_id != MLX5_DEFAULT_COPY_ID)
3014 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID ||
3015 mcp_res->refcnt == 1);
3018 /* Provide the full width of FLAG specific value. */
3019 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
3020 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
3021 /* Build a new flow. */
3022 if (mark_id != MLX5_DEFAULT_COPY_ID) {
3023 items[0] = (struct rte_flow_item){
3024 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG,
3027 items[1] = (struct rte_flow_item){
3028 .type = RTE_FLOW_ITEM_TYPE_END,
3030 actions[0] = (struct rte_flow_action){
3031 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK,
3034 actions[1] = (struct rte_flow_action){
3035 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3038 actions[2] = (struct rte_flow_action){
3039 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3042 actions[3] = (struct rte_flow_action){
3043 .type = RTE_FLOW_ACTION_TYPE_END,
3046 /* Default rule, wildcard match. */
3047 attr.priority = MLX5_FLOW_PRIO_RSVD;
3048 items[0] = (struct rte_flow_item){
3049 .type = RTE_FLOW_ITEM_TYPE_END,
3051 actions[0] = (struct rte_flow_action){
3052 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3055 actions[1] = (struct rte_flow_action){
3056 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3059 actions[2] = (struct rte_flow_action){
3060 .type = RTE_FLOW_ACTION_TYPE_END,
3063 /* Build a new entry. */
3064 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
3071 * The copy Flows are not included in any list. There
3072 * ones are referenced from other Flows and can not
3073 * be applied, removed, deleted in ardbitrary order
3074 * by list traversing.
3076 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
3077 actions, false, error);
3078 if (!mcp_res->rix_flow)
3081 mcp_res->hlist_ent.key = mark_id;
3082 ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
3083 &mcp_res->hlist_ent);
3089 if (mcp_res->rix_flow)
3090 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3091 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3096 * Release flow in RX_CP_TBL.
3099 * Pointer to Ethernet device.
3101 * Parent flow for wich copying is provided.
3104 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
3105 struct rte_flow *flow)
3107 struct mlx5_flow_mreg_copy_resource *mcp_res;
3108 struct mlx5_priv *priv = dev->data->dev_private;
3110 if (!flow->rix_mreg_copy)
3112 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3113 flow->rix_mreg_copy);
3114 if (!mcp_res || !priv->mreg_cp_tbl)
3116 if (flow->copy_applied) {
3117 MLX5_ASSERT(mcp_res->appcnt);
3118 flow->copy_applied = 0;
3120 if (!mcp_res->appcnt) {
3121 struct rte_flow *mcp_flow = mlx5_ipool_get
3122 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3126 flow_drv_remove(dev, mcp_flow);
3130 * We do not check availability of metadata registers here,
3131 * because copy resources are not allocated in this case.
3133 if (--mcp_res->refcnt)
3135 MLX5_ASSERT(mcp_res->rix_flow);
3136 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3137 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3138 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3139 flow->rix_mreg_copy = 0;
3143 * Start flow in RX_CP_TBL.
3146 * Pointer to Ethernet device.
3148 * Parent flow for wich copying is provided.
3151 * 0 on success, a negative errno value otherwise and rte_errno is set.
3154 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
3155 struct rte_flow *flow)
3157 struct mlx5_flow_mreg_copy_resource *mcp_res;
3158 struct mlx5_priv *priv = dev->data->dev_private;
3161 if (!flow->rix_mreg_copy || flow->copy_applied)
3163 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3164 flow->rix_mreg_copy);
3167 if (!mcp_res->appcnt) {
3168 struct rte_flow *mcp_flow = mlx5_ipool_get
3169 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3173 ret = flow_drv_apply(dev, mcp_flow, NULL);
3179 flow->copy_applied = 1;
3184 * Stop flow in RX_CP_TBL.
3187 * Pointer to Ethernet device.
3189 * Parent flow for wich copying is provided.
3192 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
3193 struct rte_flow *flow)
3195 struct mlx5_flow_mreg_copy_resource *mcp_res;
3196 struct mlx5_priv *priv = dev->data->dev_private;
3198 if (!flow->rix_mreg_copy || !flow->copy_applied)
3200 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
3201 flow->rix_mreg_copy);
3204 MLX5_ASSERT(mcp_res->appcnt);
3206 flow->copy_applied = 0;
3207 if (!mcp_res->appcnt) {
3208 struct rte_flow *mcp_flow = mlx5_ipool_get
3209 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
3213 flow_drv_remove(dev, mcp_flow);
3218 * Remove the default copy action from RX_CP_TBL.
3221 * Pointer to Ethernet device.
3224 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
3226 struct mlx5_flow_mreg_copy_resource *mcp_res;
3227 struct mlx5_priv *priv = dev->data->dev_private;
3229 /* Check if default flow is registered. */
3230 if (!priv->mreg_cp_tbl)
3232 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
3233 MLX5_DEFAULT_COPY_ID);
3236 MLX5_ASSERT(mcp_res->rix_flow);
3237 flow_list_destroy(dev, NULL, mcp_res->rix_flow);
3238 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
3239 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
3243 * Add the default copy action in in RX_CP_TBL.
3246 * Pointer to Ethernet device.
3248 * Perform verbose error reporting if not NULL.
3251 * 0 for success, negative value otherwise and rte_errno is set.
3254 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
3255 struct rte_flow_error *error)
3257 struct mlx5_priv *priv = dev->data->dev_private;
3258 struct mlx5_flow_mreg_copy_resource *mcp_res;
3260 /* Check whether extensive metadata feature is engaged. */
3261 if (!priv->config.dv_flow_en ||
3262 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3263 !mlx5_flow_ext_mreg_supported(dev) ||
3264 !priv->sh->dv_regc0_mask)
3266 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
3273 * Add a flow of copying flow metadata registers in RX_CP_TBL.
3275 * All the flow having Q/RSS action should be split by
3276 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
3277 * performs the following,
3278 * - CQE->flow_tag := reg_c[1] (MARK)
3279 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3280 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
3281 * but there should be a flow per each MARK ID set by MARK action.
3283 * For the aforementioned reason, if there's a MARK action in flow's action
3284 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
3285 * the MARK ID to CQE's flow_tag like,
3286 * - If reg_c[1] is mark_id,
3287 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3289 * For SET_META action which stores value in reg_c[0], as the destination is
3290 * also a flow metadata register (reg_b), adding a default flow is enough. Zero
3291 * MARK ID means the default flow. The default flow looks like,
3292 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
3295 * Pointer to Ethernet device.
3297 * Pointer to flow structure.
3298 * @param[in] actions
3299 * Pointer to the list of actions.
3301 * Perform verbose error reporting if not NULL.
3304 * 0 on success, negative value otherwise and rte_errno is set.
3307 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
3308 struct rte_flow *flow,
3309 const struct rte_flow_action *actions,
3310 struct rte_flow_error *error)
3312 struct mlx5_priv *priv = dev->data->dev_private;
3313 struct mlx5_dev_config *config = &priv->config;
3314 struct mlx5_flow_mreg_copy_resource *mcp_res;
3315 const struct rte_flow_action_mark *mark;
3317 /* Check whether extensive metadata feature is engaged. */
3318 if (!config->dv_flow_en ||
3319 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3320 !mlx5_flow_ext_mreg_supported(dev) ||
3321 !priv->sh->dv_regc0_mask)
3323 /* Find MARK action. */
3324 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3325 switch (actions->type) {
3326 case RTE_FLOW_ACTION_TYPE_FLAG:
3327 mcp_res = flow_mreg_add_copy_action
3328 (dev, MLX5_FLOW_MARK_DEFAULT, error);
3331 flow->rix_mreg_copy = mcp_res->idx;
3332 if (dev->data->dev_started) {
3334 flow->copy_applied = 1;
3337 case RTE_FLOW_ACTION_TYPE_MARK:
3338 mark = (const struct rte_flow_action_mark *)
3341 flow_mreg_add_copy_action(dev, mark->id, error);
3344 flow->rix_mreg_copy = mcp_res->idx;
3345 if (dev->data->dev_started) {
3347 flow->copy_applied = 1;
3357 #define MLX5_MAX_SPLIT_ACTIONS 24
3358 #define MLX5_MAX_SPLIT_ITEMS 24
3361 * Split the hairpin flow.
3362 * Since HW can't support encap on Rx we move the encap to Tx.
3363 * If the count action is after the encap then we also
3364 * move the count action. in this case the count will also measure
3368 * Pointer to Ethernet device.
3369 * @param[in] actions
3370 * Associated actions (list terminated by the END action).
3371 * @param[out] actions_rx
3373 * @param[out] actions_tx
3375 * @param[out] pattern_tx
3376 * The pattern items for the Tx flow.
3377 * @param[out] flow_id
3378 * The flow ID connected to this flow.
3384 flow_hairpin_split(struct rte_eth_dev *dev,
3385 const struct rte_flow_action actions[],
3386 struct rte_flow_action actions_rx[],
3387 struct rte_flow_action actions_tx[],
3388 struct rte_flow_item pattern_tx[],
3391 struct mlx5_priv *priv = dev->data->dev_private;
3392 const struct rte_flow_action_raw_encap *raw_encap;
3393 const struct rte_flow_action_raw_decap *raw_decap;
3394 struct mlx5_rte_flow_action_set_tag *set_tag;
3395 struct rte_flow_action *tag_action;
3396 struct mlx5_rte_flow_item_tag *tag_item;
3397 struct rte_flow_item *item;
3401 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
3402 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3403 switch (actions->type) {
3404 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3405 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3406 rte_memcpy(actions_tx, actions,
3407 sizeof(struct rte_flow_action));
3410 case RTE_FLOW_ACTION_TYPE_COUNT:
3412 rte_memcpy(actions_tx, actions,
3413 sizeof(struct rte_flow_action));
3416 rte_memcpy(actions_rx, actions,
3417 sizeof(struct rte_flow_action));
3421 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3422 raw_encap = actions->conf;
3423 if (raw_encap->size >
3424 (sizeof(struct rte_flow_item_eth) +
3425 sizeof(struct rte_flow_item_ipv4))) {
3426 memcpy(actions_tx, actions,
3427 sizeof(struct rte_flow_action));
3431 rte_memcpy(actions_rx, actions,
3432 sizeof(struct rte_flow_action));
3436 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3437 raw_decap = actions->conf;
3438 if (raw_decap->size <
3439 (sizeof(struct rte_flow_item_eth) +
3440 sizeof(struct rte_flow_item_ipv4))) {
3441 memcpy(actions_tx, actions,
3442 sizeof(struct rte_flow_action));
3445 rte_memcpy(actions_rx, actions,
3446 sizeof(struct rte_flow_action));
3451 rte_memcpy(actions_rx, actions,
3452 sizeof(struct rte_flow_action));
3457 /* Add set meta action and end action for the Rx flow. */
3458 tag_action = actions_rx;
3459 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3461 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
3463 set_tag = (void *)actions_rx;
3464 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
3465 MLX5_ASSERT(set_tag->id > REG_NONE);
3466 set_tag->data = *flow_id;
3467 tag_action->conf = set_tag;
3468 /* Create Tx item list. */
3469 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
3470 addr = (void *)&pattern_tx[2];
3472 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG;
3473 tag_item = (void *)addr;
3474 tag_item->data = *flow_id;
3475 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
3476 MLX5_ASSERT(set_tag->id > REG_NONE);
3477 item->spec = tag_item;
3478 addr += sizeof(struct mlx5_rte_flow_item_tag);
3479 tag_item = (void *)addr;
3480 tag_item->data = UINT32_MAX;
3481 tag_item->id = UINT16_MAX;
3482 item->mask = tag_item;
3483 addr += sizeof(struct mlx5_rte_flow_item_tag);
3486 item->type = RTE_FLOW_ITEM_TYPE_END;
3491 * The last stage of splitting chain, just creates the subflow
3492 * without any modification.
3495 * Pointer to Ethernet device.
3497 * Parent flow structure pointer.
3498 * @param[in, out] sub_flow
3499 * Pointer to return the created subflow, may be NULL.
3500 * @param[in] prefix_layers
3501 * Prefix subflow layers, may be 0.
3503 * Flow rule attributes.
3505 * Pattern specification (list terminated by the END pattern item).
3506 * @param[in] actions
3507 * Associated actions (list terminated by the END action).
3508 * @param[in] external
3509 * This flow rule is created by request external to PMD.
3510 * @param[in] flow_idx
3511 * This memory pool index to the flow.
3513 * Perform verbose error reporting if not NULL.
3515 * 0 on success, negative value otherwise
3518 flow_create_split_inner(struct rte_eth_dev *dev,
3519 struct rte_flow *flow,
3520 struct mlx5_flow **sub_flow,
3521 uint64_t prefix_layers,
3522 const struct rte_flow_attr *attr,
3523 const struct rte_flow_item items[],
3524 const struct rte_flow_action actions[],
3525 bool external, uint32_t flow_idx,
3526 struct rte_flow_error *error)
3528 struct mlx5_flow *dev_flow;
3530 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
3534 dev_flow->flow = flow;
3535 dev_flow->external = external;
3536 /* Subflow object was created, we must include one in the list. */
3537 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
3538 dev_flow->handle, next);
3540 * If dev_flow is as one of the suffix flow, some actions in suffix
3541 * flow may need some user defined item layer flags.
3544 dev_flow->handle->layers = prefix_layers;
3546 *sub_flow = dev_flow;
3547 return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
3551 * Split the meter flow.
3553 * As meter flow will split to three sub flow, other than meter
3554 * action, the other actions make sense to only meter accepts
3555 * the packet. If it need to be dropped, no other additional
3556 * actions should be take.
3558 * One kind of special action which decapsulates the L3 tunnel
3559 * header will be in the prefix sub flow, as not to take the
3560 * L3 tunnel header into account.
3563 * Pointer to Ethernet device.
3565 * Pattern specification (list terminated by the END pattern item).
3566 * @param[out] sfx_items
3567 * Suffix flow match items (list terminated by the END pattern item).
3568 * @param[in] actions
3569 * Associated actions (list terminated by the END action).
3570 * @param[out] actions_sfx
3571 * Suffix flow actions.
3572 * @param[out] actions_pre
3573 * Prefix flow actions.
3574 * @param[out] pattern_sfx
3575 * The pattern items for the suffix flow.
3576 * @param[out] tag_sfx
3577 * Pointer to suffix flow tag.
3583 flow_meter_split_prep(struct rte_eth_dev *dev,
3584 const struct rte_flow_item items[],
3585 struct rte_flow_item sfx_items[],
3586 const struct rte_flow_action actions[],
3587 struct rte_flow_action actions_sfx[],
3588 struct rte_flow_action actions_pre[])
3590 struct rte_flow_action *tag_action = NULL;
3591 struct rte_flow_item *tag_item;
3592 struct mlx5_rte_flow_action_set_tag *set_tag;
3593 struct rte_flow_error error;
3594 const struct rte_flow_action_raw_encap *raw_encap;
3595 const struct rte_flow_action_raw_decap *raw_decap;
3596 struct mlx5_rte_flow_item_tag *tag_spec;
3597 struct mlx5_rte_flow_item_tag *tag_mask;
3599 bool copy_vlan = false;
3601 /* Prepare the actions for prefix and suffix flow. */
3602 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3603 struct rte_flow_action **action_cur = NULL;
3605 switch (actions->type) {
3606 case RTE_FLOW_ACTION_TYPE_METER:
3607 /* Add the extra tag action first. */
3608 tag_action = actions_pre;
3609 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3611 action_cur = &actions_pre;
3613 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3614 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3615 action_cur = &actions_pre;
3617 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3618 raw_encap = actions->conf;
3619 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
3620 action_cur = &actions_pre;
3622 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3623 raw_decap = actions->conf;
3624 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3625 action_cur = &actions_pre;
3627 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3628 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3635 action_cur = &actions_sfx;
3636 memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
3639 /* Add end action to the actions. */
3640 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
3641 actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
3644 set_tag = (void *)actions_pre;
3645 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
3647 * Get the id from the qrss_pool to make qrss share the id with meter.
3649 tag_id = flow_qrss_get_id(dev);
3650 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
3652 tag_action->conf = set_tag;
3653 /* Prepare the suffix subflow items. */
3654 tag_item = sfx_items++;
3655 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3656 int item_type = items->type;
3658 switch (item_type) {
3659 case RTE_FLOW_ITEM_TYPE_PORT_ID:
3660 memcpy(sfx_items, items, sizeof(*sfx_items));
3663 case RTE_FLOW_ITEM_TYPE_VLAN:
3665 memcpy(sfx_items, items, sizeof(*sfx_items));
3667 * Convert to internal match item, it is used
3668 * for vlan push and set vid.
3670 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
3678 sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
3680 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
3681 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
3682 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
3683 tag_mask = tag_spec + 1;
3684 tag_mask->data = 0xffffff00;
3685 tag_item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG;
3686 tag_item->spec = tag_spec;
3687 tag_item->last = NULL;
3688 tag_item->mask = tag_mask;
3693 * Split action list having QUEUE/RSS for metadata register copy.
3695 * Once Q/RSS action is detected in user's action list, the flow action
3696 * should be split in order to copy metadata registers, which will happen in
3698 * - CQE->flow_tag := reg_c[1] (MARK)
3699 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
3700 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
3701 * This is because the last action of each flow must be a terminal action
3702 * (QUEUE, RSS or DROP).
3704 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
3705 * stored and kept in the mlx5_flow structure per each sub_flow.
3707 * The Q/RSS action is replaced with,
3708 * - SET_TAG, setting the allocated flow ID to reg_c[2].
3709 * And the following JUMP action is added at the end,
3710 * - JUMP, to RX_CP_TBL.
3712 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
3713 * flow_create_split_metadata() routine. The flow will look like,
3714 * - If flow ID matches (reg_c[2]), perform Q/RSS.
3717 * Pointer to Ethernet device.
3718 * @param[out] split_actions
3719 * Pointer to store split actions to jump to CP_TBL.
3720 * @param[in] actions
3721 * Pointer to the list of original flow actions.
3723 * Pointer to the Q/RSS action.
3724 * @param[in] actions_n
3725 * Number of original actions.
3727 * Perform verbose error reporting if not NULL.
3730 * non-zero unique flow_id on success, otherwise 0 and
3731 * error/rte_error are set.
3734 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
3735 struct rte_flow_action *split_actions,
3736 const struct rte_flow_action *actions,
3737 const struct rte_flow_action *qrss,
3738 int actions_n, struct rte_flow_error *error)
3740 struct mlx5_rte_flow_action_set_tag *set_tag;
3741 struct rte_flow_action_jump *jump;
3742 const int qrss_idx = qrss - actions;
3743 uint32_t flow_id = 0;
3747 * Given actions will be split
3748 * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
3749 * - Add jump to mreg CP_TBL.
3750 * As a result, there will be one more action.
3753 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
3754 set_tag = (void *)(split_actions + actions_n);
3756 * If tag action is not set to void(it means we are not the meter
3757 * suffix flow), add the tag action. Since meter suffix flow already
3758 * has the tag added.
3760 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
3762 * Allocate the new subflow ID. This one is unique within
3763 * device and not shared with representors. Otherwise,
3764 * we would have to resolve multi-thread access synch
3765 * issue. Each flow on the shared device is appended
3766 * with source vport identifier, so the resulting
3767 * flows will be unique in the shared (by master and
3768 * representors) domain even if they have coinciding
3771 flow_id = flow_qrss_get_id(dev);
3773 return rte_flow_error_set(error, ENOMEM,
3774 RTE_FLOW_ERROR_TYPE_ACTION,
3775 NULL, "can't allocate id "
3776 "for split Q/RSS subflow");
3777 /* Internal SET_TAG action to set flow ID. */
3778 *set_tag = (struct mlx5_rte_flow_action_set_tag){
3781 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
3785 /* Construct new actions array. */
3786 /* Replace QUEUE/RSS action. */
3787 split_actions[qrss_idx] = (struct rte_flow_action){
3788 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG,
3792 /* JUMP action to jump to mreg copy table (CP_TBL). */
3793 jump = (void *)(set_tag + 1);
3794 *jump = (struct rte_flow_action_jump){
3795 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
3797 split_actions[actions_n - 2] = (struct rte_flow_action){
3798 .type = RTE_FLOW_ACTION_TYPE_JUMP,
3801 split_actions[actions_n - 1] = (struct rte_flow_action){
3802 .type = RTE_FLOW_ACTION_TYPE_END,
3808 * Extend the given action list for Tx metadata copy.
3810 * Copy the given action list to the ext_actions and add flow metadata register
3811 * copy action in order to copy reg_a set by WQE to reg_c[0].
3813 * @param[out] ext_actions
3814 * Pointer to the extended action list.
3815 * @param[in] actions
3816 * Pointer to the list of actions.
3817 * @param[in] actions_n
3818 * Number of actions in the list.
3820 * Perform verbose error reporting if not NULL.
3821 * @param[in] encap_idx
3822 * The encap action inndex.
3825 * 0 on success, negative value otherwise
3828 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
3829 struct rte_flow_action *ext_actions,
3830 const struct rte_flow_action *actions,
3831 int actions_n, struct rte_flow_error *error,
3834 struct mlx5_flow_action_copy_mreg *cp_mreg =
3835 (struct mlx5_flow_action_copy_mreg *)
3836 (ext_actions + actions_n + 1);
3839 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
3843 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
3848 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
3849 if (encap_idx == actions_n - 1) {
3850 ext_actions[actions_n - 1] = (struct rte_flow_action){
3851 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3854 ext_actions[actions_n] = (struct rte_flow_action){
3855 .type = RTE_FLOW_ACTION_TYPE_END,
3858 ext_actions[encap_idx] = (struct rte_flow_action){
3859 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
3862 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
3863 sizeof(*ext_actions) * (actions_n - encap_idx));
3869 * The splitting for metadata feature.
3871 * - Q/RSS action on NIC Rx should be split in order to pass by
3872 * the mreg copy table (RX_CP_TBL) and then it jumps to the
3873 * action table (RX_ACT_TBL) which has the split Q/RSS action.
3875 * - All the actions on NIC Tx should have a mreg copy action to
3876 * copy reg_a from WQE to reg_c[0].
3879 * Pointer to Ethernet device.
3881 * Parent flow structure pointer.
3882 * @param[in] prefix_layers
3883 * Prefix flow layer flags.
3885 * Flow rule attributes.
3887 * Pattern specification (list terminated by the END pattern item).
3888 * @param[in] actions
3889 * Associated actions (list terminated by the END action).
3890 * @param[in] external
3891 * This flow rule is created by request external to PMD.
3892 * @param[in] flow_idx
3893 * This memory pool index to the flow.
3895 * Perform verbose error reporting if not NULL.
3897 * 0 on success, negative value otherwise
3900 flow_create_split_metadata(struct rte_eth_dev *dev,
3901 struct rte_flow *flow,
3902 uint64_t prefix_layers,
3903 const struct rte_flow_attr *attr,
3904 const struct rte_flow_item items[],
3905 const struct rte_flow_action actions[],
3906 bool external, uint32_t flow_idx,
3907 struct rte_flow_error *error)
3909 struct mlx5_priv *priv = dev->data->dev_private;
3910 struct mlx5_dev_config *config = &priv->config;
3911 const struct rte_flow_action *qrss = NULL;
3912 struct rte_flow_action *ext_actions = NULL;
3913 struct mlx5_flow *dev_flow = NULL;
3914 uint32_t qrss_id = 0;
3921 /* Check whether extensive metadata feature is engaged. */
3922 if (!config->dv_flow_en ||
3923 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
3924 !mlx5_flow_ext_mreg_supported(dev))
3925 return flow_create_split_inner(dev, flow, NULL, prefix_layers,
3926 attr, items, actions, external,
3928 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
3931 /* Exclude hairpin flows from splitting. */
3932 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
3933 const struct rte_flow_action_queue *queue;
3936 if (mlx5_rxq_get_type(dev, queue->index) ==
3937 MLX5_RXQ_TYPE_HAIRPIN)
3939 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
3940 const struct rte_flow_action_rss *rss;
3943 if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
3944 MLX5_RXQ_TYPE_HAIRPIN)
3949 /* Check if it is in meter suffix table. */
3950 mtr_sfx = attr->group == (attr->transfer ?
3951 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
3952 MLX5_FLOW_TABLE_LEVEL_SUFFIX);
3954 * Q/RSS action on NIC Rx should be split in order to pass by
3955 * the mreg copy table (RX_CP_TBL) and then it jumps to the
3956 * action table (RX_ACT_TBL) which has the split Q/RSS action.
3958 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
3959 sizeof(struct rte_flow_action_set_tag) +
3960 sizeof(struct rte_flow_action_jump);
3961 ext_actions = rte_zmalloc(__func__, act_size, 0);
3963 return rte_flow_error_set(error, ENOMEM,
3964 RTE_FLOW_ERROR_TYPE_ACTION,
3965 NULL, "no memory to split "
3968 * If we are the suffix flow of meter, tag already exist.
3969 * Set the tag action to void.
3972 ext_actions[qrss - actions].type =
3973 RTE_FLOW_ACTION_TYPE_VOID;
3975 ext_actions[qrss - actions].type =
3976 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
3978 * Create the new actions list with removed Q/RSS action
3979 * and appended set tag and jump to register copy table
3980 * (RX_CP_TBL). We should preallocate unique tag ID here
3981 * in advance, because it is needed for set tag action.
3983 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
3984 qrss, actions_n, error);
3985 if (!mtr_sfx && !qrss_id) {
3989 } else if (attr->egress && !attr->transfer) {
3991 * All the actions on NIC Tx should have a metadata register
3992 * copy action to copy reg_a from WQE to reg_c[meta]
3994 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
3995 sizeof(struct mlx5_flow_action_copy_mreg);
3996 ext_actions = rte_zmalloc(__func__, act_size, 0);
3998 return rte_flow_error_set(error, ENOMEM,
3999 RTE_FLOW_ERROR_TYPE_ACTION,
4000 NULL, "no memory to split "
4002 /* Create the action list appended with copy register. */
4003 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
4004 actions_n, error, encap_idx);
4008 /* Add the unmodified original or prefix subflow. */
4009 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr,
4010 items, ext_actions ? ext_actions :
4011 actions, external, flow_idx, error);
4014 MLX5_ASSERT(dev_flow);
4016 const struct rte_flow_attr q_attr = {
4017 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4020 /* Internal PMD action to set register. */
4021 struct mlx5_rte_flow_item_tag q_tag_spec = {
4025 struct rte_flow_item q_items[] = {
4027 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4028 .spec = &q_tag_spec,
4033 .type = RTE_FLOW_ITEM_TYPE_END,
4036 struct rte_flow_action q_actions[] = {
4042 .type = RTE_FLOW_ACTION_TYPE_END,
4045 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
4048 * Configure the tag item only if there is no meter subflow.
4049 * Since tag is already marked in the meter suffix subflow
4050 * we can just use the meter suffix items as is.
4053 /* Not meter subflow. */
4054 MLX5_ASSERT(!mtr_sfx);
4056 * Put unique id in prefix flow due to it is destroyed
4057 * after suffix flow and id will be freed after there
4058 * is no actual flows with this id and identifier
4059 * reallocation becomes possible (for example, for
4060 * other flows in other threads).
4062 dev_flow->handle->split_flow_id = qrss_id;
4063 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
4067 q_tag_spec.id = ret;
4070 /* Add suffix subflow to execute Q/RSS. */
4071 ret = flow_create_split_inner(dev, flow, &dev_flow, layers,
4072 &q_attr, mtr_sfx ? items :
4074 external, flow_idx, error);
4077 /* qrss ID should be freed if failed. */
4079 MLX5_ASSERT(dev_flow);
4084 * We do not destroy the partially created sub_flows in case of error.
4085 * These ones are included into parent flow list and will be destroyed
4086 * by flow_drv_destroy.
4088 flow_qrss_free_id(dev, qrss_id);
4089 rte_free(ext_actions);
4094 * The splitting for meter feature.
4096 * - The meter flow will be split to two flows as prefix and
4097 * suffix flow. The packets make sense only it pass the prefix
4100 * - Reg_C_5 is used for the packet to match betweend prefix and
4104 * Pointer to Ethernet device.
4106 * Parent flow structure pointer.
4108 * Flow rule attributes.
4110 * Pattern specification (list terminated by the END pattern item).
4111 * @param[in] actions
4112 * Associated actions (list terminated by the END action).
4113 * @param[in] external
4114 * This flow rule is created by request external to PMD.
4115 * @param[in] flow_idx
4116 * This memory pool index to the flow.
4118 * Perform verbose error reporting if not NULL.
4120 * 0 on success, negative value otherwise
4123 flow_create_split_meter(struct rte_eth_dev *dev,
4124 struct rte_flow *flow,
4125 const struct rte_flow_attr *attr,
4126 const struct rte_flow_item items[],
4127 const struct rte_flow_action actions[],
4128 bool external, uint32_t flow_idx,
4129 struct rte_flow_error *error)
4131 struct mlx5_priv *priv = dev->data->dev_private;
4132 struct rte_flow_action *sfx_actions = NULL;
4133 struct rte_flow_action *pre_actions = NULL;
4134 struct rte_flow_item *sfx_items = NULL;
4135 struct mlx5_flow *dev_flow = NULL;
4136 struct rte_flow_attr sfx_attr = *attr;
4138 uint32_t mtr_tag_id = 0;
4145 actions_n = flow_check_meter_action(actions, &mtr);
4147 /* The five prefix actions: meter, decap, encap, tag, end. */
4148 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
4149 sizeof(struct mlx5_rte_flow_action_set_tag);
4150 /* tag, vlan, port id, end. */
4151 #define METER_SUFFIX_ITEM 4
4152 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
4153 sizeof(struct mlx5_rte_flow_item_tag) * 2;
4154 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0);
4156 return rte_flow_error_set(error, ENOMEM,
4157 RTE_FLOW_ERROR_TYPE_ACTION,
4158 NULL, "no memory to split "
4160 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
4162 pre_actions = sfx_actions + actions_n;
4163 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
4164 actions, sfx_actions,
4170 /* Add the prefix subflow. */
4171 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr,
4172 items, pre_actions, external,
4178 dev_flow->handle->split_flow_id = mtr_tag_id;
4179 /* Setting the sfx group atrr. */
4180 sfx_attr.group = sfx_attr.transfer ?
4181 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
4182 MLX5_FLOW_TABLE_LEVEL_SUFFIX;
4184 /* Add the prefix subflow. */
4185 ret = flow_create_split_metadata(dev, flow, dev_flow ?
4186 flow_get_prefix_layer_flags(dev_flow) :
4188 sfx_items ? sfx_items : items,
4189 sfx_actions ? sfx_actions : actions,
4190 external, flow_idx, error);
4193 rte_free(sfx_actions);
4198 * Split the flow to subflow set. The splitters might be linked
4199 * in the chain, like this:
4200 * flow_create_split_outer() calls:
4201 * flow_create_split_meter() calls:
4202 * flow_create_split_metadata(meter_subflow_0) calls:
4203 * flow_create_split_inner(metadata_subflow_0)
4204 * flow_create_split_inner(metadata_subflow_1)
4205 * flow_create_split_inner(metadata_subflow_2)
4206 * flow_create_split_metadata(meter_subflow_1) calls:
4207 * flow_create_split_inner(metadata_subflow_0)
4208 * flow_create_split_inner(metadata_subflow_1)
4209 * flow_create_split_inner(metadata_subflow_2)
4211 * This provide flexible way to add new levels of flow splitting.
4212 * The all of successfully created subflows are included to the
4213 * parent flow dev_flow list.
4216 * Pointer to Ethernet device.
4218 * Parent flow structure pointer.
4220 * Flow rule attributes.
4222 * Pattern specification (list terminated by the END pattern item).
4223 * @param[in] actions
4224 * Associated actions (list terminated by the END action).
4225 * @param[in] external
4226 * This flow rule is created by request external to PMD.
4227 * @param[in] flow_idx
4228 * This memory pool index to the flow.
4230 * Perform verbose error reporting if not NULL.
4232 * 0 on success, negative value otherwise
4235 flow_create_split_outer(struct rte_eth_dev *dev,
4236 struct rte_flow *flow,
4237 const struct rte_flow_attr *attr,
4238 const struct rte_flow_item items[],
4239 const struct rte_flow_action actions[],
4240 bool external, uint32_t flow_idx,
4241 struct rte_flow_error *error)
4245 ret = flow_create_split_meter(dev, flow, attr, items,
4246 actions, external, flow_idx, error);
4247 MLX5_ASSERT(ret <= 0);
4252 * Create a flow and add it to @p list.
4255 * Pointer to Ethernet device.
4257 * Pointer to a TAILQ flow list. If this parameter NULL,
4258 * no list insertion occurred, flow is just created,
4259 * this is caller's responsibility to track the
4262 * Flow rule attributes.
4264 * Pattern specification (list terminated by the END pattern item).
4265 * @param[in] actions
4266 * Associated actions (list terminated by the END action).
4267 * @param[in] external
4268 * This flow rule is created by request external to PMD.
4270 * Perform verbose error reporting if not NULL.
4273 * A flow index on success, 0 otherwise and rte_errno is set.
4276 flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
4277 const struct rte_flow_attr *attr,
4278 const struct rte_flow_item items[],
4279 const struct rte_flow_action actions[],
4280 bool external, struct rte_flow_error *error)
4282 struct mlx5_priv *priv = dev->data->dev_private;
4283 struct rte_flow *flow = NULL;
4284 struct mlx5_flow *dev_flow;
4285 const struct rte_flow_action_rss *rss;
4287 struct rte_flow_expand_rss buf;
4288 uint8_t buffer[2048];
4291 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4292 uint8_t buffer[2048];
4295 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
4296 uint8_t buffer[2048];
4297 } actions_hairpin_tx;
4299 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
4300 uint8_t buffer[2048];
4302 struct rte_flow_expand_rss *buf = &expand_buffer.buf;
4303 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
4304 priv->rss_desc)[!!priv->flow_idx];
4305 const struct rte_flow_action *p_actions_rx = actions;
4309 uint32_t hairpin_id = 0;
4310 struct rte_flow_attr attr_tx = { .priority = 0 };
4313 hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4314 ret = flow_drv_validate(dev, attr, items, p_actions_rx,
4315 external, hairpin_flow, error);
4318 if (hairpin_flow > 0) {
4319 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
4323 flow_hairpin_split(dev, actions, actions_rx.actions,
4324 actions_hairpin_tx.actions, items_tx.items,
4326 p_actions_rx = actions_rx.actions;
4328 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
4331 goto error_before_flow;
4333 flow->drv_type = flow_get_drv_type(dev, attr);
4334 if (hairpin_id != 0)
4335 flow->hairpin_flow_id = hairpin_id;
4336 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
4337 flow->drv_type < MLX5_FLOW_TYPE_MAX);
4338 memset(rss_desc, 0, sizeof(*rss_desc));
4339 rss = flow_get_rss_action(p_actions_rx);
4342 * The following information is required by
4343 * mlx5_flow_hashfields_adjust() in advance.
4345 rss_desc->level = rss->level;
4346 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
4347 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
4349 flow->dev_handles = 0;
4350 if (rss && rss->types) {
4351 unsigned int graph_root;
4353 graph_root = find_graph_root(items, rss->level);
4354 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
4356 mlx5_support_expansion,
4358 MLX5_ASSERT(ret > 0 &&
4359 (unsigned int)ret < sizeof(expand_buffer.buffer));
4362 buf->entry[0].pattern = (void *)(uintptr_t)items;
4365 * Record the start index when there is a nested call. All sub-flows
4366 * need to be translated before another calling.
4367 * No need to use ping-pong buffer to save memory here.
4369 if (priv->flow_idx) {
4370 MLX5_ASSERT(!priv->flow_nested_idx);
4371 priv->flow_nested_idx = priv->flow_idx;
4373 for (i = 0; i < buf->entries; ++i) {
4375 * The splitter may create multiple dev_flows,
4376 * depending on configuration. In the simplest
4377 * case it just creates unmodified original flow.
4379 ret = flow_create_split_outer(dev, flow, attr,
4380 buf->entry[i].pattern,
4381 p_actions_rx, external, idx,
4386 /* Create the tx flow. */
4388 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
4389 attr_tx.ingress = 0;
4391 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
4392 actions_hairpin_tx.actions,
4396 dev_flow->flow = flow;
4397 dev_flow->external = 0;
4398 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4399 dev_flow->handle, next);
4400 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
4402 actions_hairpin_tx.actions, error);
4407 * Update the metadata register copy table. If extensive
4408 * metadata feature is enabled and registers are supported
4409 * we might create the extra rte_flow for each unique
4410 * MARK/FLAG action ID.
4412 * The table is updated for ingress Flows only, because
4413 * the egress Flows belong to the different device and
4414 * copy table should be updated in peer NIC Rx domain.
4416 if (attr->ingress &&
4417 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
4418 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
4423 * If the flow is external (from application) OR device is started, then
4424 * the flow will be applied immediately.
4426 if (external || dev->data->dev_started) {
4427 ret = flow_drv_apply(dev, flow, error);
4432 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
4434 flow_rxq_flags_set(dev, flow);
4435 /* Nested flow creation index recovery. */
4436 priv->flow_idx = priv->flow_nested_idx;
4437 if (priv->flow_nested_idx)
4438 priv->flow_nested_idx = 0;
4442 ret = rte_errno; /* Save rte_errno before cleanup. */
4443 flow_mreg_del_copy_action(dev, flow);
4444 flow_drv_destroy(dev, flow);
4445 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
4446 rte_errno = ret; /* Restore rte_errno. */
4450 mlx5_flow_id_release(priv->sh->flow_id_pool,
4453 priv->flow_idx = priv->flow_nested_idx;
4454 if (priv->flow_nested_idx)
4455 priv->flow_nested_idx = 0;
4460 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
4461 * incoming packets to table 1.
4463 * Other flow rules, requested for group n, will be created in
4464 * e-switch table n+1.
4465 * Jump action to e-switch group n will be created to group n+1.
4467 * Used when working in switchdev mode, to utilise advantages of table 1
4471 * Pointer to Ethernet device.
4474 * Pointer to flow on success, NULL otherwise and rte_errno is set.
4477 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
4479 const struct rte_flow_attr attr = {
4486 const struct rte_flow_item pattern = {
4487 .type = RTE_FLOW_ITEM_TYPE_END,
4489 struct rte_flow_action_jump jump = {
4492 const struct rte_flow_action actions[] = {
4494 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4498 .type = RTE_FLOW_ACTION_TYPE_END,
4501 struct mlx5_priv *priv = dev->data->dev_private;
4502 struct rte_flow_error error;
4504 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
4506 actions, false, &error);
4510 * Validate a flow supported by the NIC.
4512 * @see rte_flow_validate()
4516 mlx5_flow_validate(struct rte_eth_dev *dev,
4517 const struct rte_flow_attr *attr,
4518 const struct rte_flow_item items[],
4519 const struct rte_flow_action actions[],
4520 struct rte_flow_error *error)
4524 hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
4525 return flow_drv_validate(dev, attr, items, actions,
4526 true, hairpin_flow, error);
4532 * @see rte_flow_create()
4536 mlx5_flow_create(struct rte_eth_dev *dev,
4537 const struct rte_flow_attr *attr,
4538 const struct rte_flow_item items[],
4539 const struct rte_flow_action actions[],
4540 struct rte_flow_error *error)
4542 struct mlx5_priv *priv = dev->data->dev_private;
4545 * If the device is not started yet, it is not allowed to created a
4546 * flow from application. PMD default flows and traffic control flows
4549 if (unlikely(!dev->data->dev_started)) {
4550 DRV_LOG(DEBUG, "port %u is not started when "
4551 "inserting a flow", dev->data->port_id);
4552 rte_flow_error_set(error, ENODEV,
4553 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4555 "port not started");
4558 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
4559 attr, items, actions, true, error);
4563 * Destroy a flow in a list.
4566 * Pointer to Ethernet device.
4568 * Pointer to the Indexed flow list. If this parameter NULL,
4569 * there is no flow removal from the list. Be noted that as
4570 * flow is add to the indexed list, memory of the indexed
4571 * list points to maybe changed as flow destroyed.
4572 * @param[in] flow_idx
4573 * Index of flow to destroy.
4576 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
4579 struct mlx5_priv *priv = dev->data->dev_private;
4580 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
4581 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
4582 [MLX5_IPOOL_RTE_FLOW], flow_idx);
4587 * Update RX queue flags only if port is started, otherwise it is
4590 if (dev->data->dev_started)
4591 flow_rxq_flags_trim(dev, flow);
4592 if (flow->hairpin_flow_id)
4593 mlx5_flow_id_release(priv->sh->flow_id_pool,
4594 flow->hairpin_flow_id);
4595 flow_drv_destroy(dev, flow);
4597 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
4598 flow_idx, flow, next);
4599 flow_mreg_del_copy_action(dev, flow);
4601 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
4602 if (priv_fdir_flow->rix_flow == flow_idx)
4605 if (priv_fdir_flow) {
4606 LIST_REMOVE(priv_fdir_flow, next);
4607 rte_free(priv_fdir_flow->fdir);
4608 rte_free(priv_fdir_flow);
4611 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
4615 * Destroy all flows.
4618 * Pointer to Ethernet device.
4620 * Pointer to the Indexed flow list.
4622 * If flushing is called avtively.
4625 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
4627 uint32_t num_flushed = 0;
4630 flow_list_destroy(dev, list, *list);
4634 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
4635 dev->data->port_id, num_flushed);
4643 * Pointer to Ethernet device.
4645 * Pointer to the Indexed flow list.
4648 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list)
4650 struct mlx5_priv *priv = dev->data->dev_private;
4651 struct rte_flow *flow = NULL;
4654 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
4656 flow_drv_remove(dev, flow);
4657 flow_mreg_stop_copy_action(dev, flow);
4659 flow_mreg_del_default_copy_action(dev);
4660 flow_rxq_flags_clear(dev);
4667 * Pointer to Ethernet device.
4669 * Pointer to the Indexed flow list.
4672 * 0 on success, a negative errno value otherwise and rte_errno is set.
4675 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list)
4677 struct mlx5_priv *priv = dev->data->dev_private;
4678 struct rte_flow *flow = NULL;
4679 struct rte_flow_error error;
4683 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
4684 ret = flow_mreg_add_default_copy_action(dev, &error);
4687 /* Apply Flows created by application. */
4688 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx,
4690 ret = flow_mreg_start_copy_action(dev, flow);
4693 ret = flow_drv_apply(dev, flow, &error);
4696 flow_rxq_flags_set(dev, flow);
4700 ret = rte_errno; /* Save rte_errno before cleanup. */
4701 mlx5_flow_stop(dev, list);
4702 rte_errno = ret; /* Restore rte_errno. */
4707 * Stop all default actions for flows.
4710 * Pointer to Ethernet device.
4713 mlx5_flow_stop_default(struct rte_eth_dev *dev)
4715 flow_mreg_del_default_copy_action(dev);
4716 flow_rxq_flags_clear(dev);
4720 * Start all default actions for flows.
4723 * Pointer to Ethernet device.
4725 * 0 on success, a negative errno value otherwise and rte_errno is set.
4728 mlx5_flow_start_default(struct rte_eth_dev *dev)
4730 struct rte_flow_error error;
4732 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
4733 return flow_mreg_add_default_copy_action(dev, &error);
4737 * Allocate intermediate resources for flow creation.
4740 * Pointer to Ethernet device.
4743 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev)
4745 struct mlx5_priv *priv = dev->data->dev_private;
4747 if (!priv->inter_flows) {
4748 priv->inter_flows = rte_calloc(__func__, 1,
4749 MLX5_NUM_MAX_DEV_FLOWS *
4750 sizeof(struct mlx5_flow) +
4751 (sizeof(struct mlx5_flow_rss_desc) +
4752 sizeof(uint16_t) * UINT16_MAX) * 2, 0);
4753 if (!priv->inter_flows) {
4754 DRV_LOG(ERR, "can't allocate intermediate memory.");
4758 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows)
4759 [MLX5_NUM_MAX_DEV_FLOWS];
4760 /* Reset the index. */
4762 priv->flow_nested_idx = 0;
4766 * Free intermediate resources for flows.
4769 * Pointer to Ethernet device.
4772 mlx5_flow_free_intermediate(struct rte_eth_dev *dev)
4774 struct mlx5_priv *priv = dev->data->dev_private;
4776 rte_free(priv->inter_flows);
4777 priv->inter_flows = NULL;
4781 * Verify the flow list is empty
4784 * Pointer to Ethernet device.
4786 * @return the number of flows not released.
4789 mlx5_flow_verify(struct rte_eth_dev *dev)
4791 struct mlx5_priv *priv = dev->data->dev_private;
4792 struct rte_flow *flow;
4796 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
4798 DRV_LOG(DEBUG, "port %u flow %p still referenced",
4799 dev->data->port_id, (void *)flow);
4806 * Enable default hairpin egress flow.
4809 * Pointer to Ethernet device.
4814 * 0 on success, a negative errno value otherwise and rte_errno is set.
4817 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
4820 struct mlx5_priv *priv = dev->data->dev_private;
4821 const struct rte_flow_attr attr = {
4825 struct mlx5_rte_flow_item_tx_queue queue_spec = {
4828 struct mlx5_rte_flow_item_tx_queue queue_mask = {
4829 .queue = UINT32_MAX,
4831 struct rte_flow_item items[] = {
4833 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
4834 .spec = &queue_spec,
4836 .mask = &queue_mask,
4839 .type = RTE_FLOW_ITEM_TYPE_END,
4842 struct rte_flow_action_jump jump = {
4843 .group = MLX5_HAIRPIN_TX_TABLE,
4845 struct rte_flow_action actions[2];
4847 struct rte_flow_error error;
4849 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
4850 actions[0].conf = &jump;
4851 actions[1].type = RTE_FLOW_ACTION_TYPE_END;
4852 flow_idx = flow_list_create(dev, &priv->ctrl_flows,
4853 &attr, items, actions, false, &error);
4856 "Failed to create ctrl flow: rte_errno(%d),"
4857 " type(%d), message(%s)",
4858 rte_errno, error.type,
4859 error.message ? error.message : " (no stated reason)");
4866 * Enable a control flow configured from the control plane.
4869 * Pointer to Ethernet device.
4871 * An Ethernet flow spec to apply.
4873 * An Ethernet flow mask to apply.
4875 * A VLAN flow spec to apply.
4877 * A VLAN flow mask to apply.
4880 * 0 on success, a negative errno value otherwise and rte_errno is set.
4883 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
4884 struct rte_flow_item_eth *eth_spec,
4885 struct rte_flow_item_eth *eth_mask,
4886 struct rte_flow_item_vlan *vlan_spec,
4887 struct rte_flow_item_vlan *vlan_mask)
4889 struct mlx5_priv *priv = dev->data->dev_private;
4890 const struct rte_flow_attr attr = {
4892 .priority = MLX5_FLOW_PRIO_RSVD,
4894 struct rte_flow_item items[] = {
4896 .type = RTE_FLOW_ITEM_TYPE_ETH,
4902 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
4903 RTE_FLOW_ITEM_TYPE_END,
4909 .type = RTE_FLOW_ITEM_TYPE_END,
4912 uint16_t queue[priv->reta_idx_n];
4913 struct rte_flow_action_rss action_rss = {
4914 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
4916 .types = priv->rss_conf.rss_hf,
4917 .key_len = priv->rss_conf.rss_key_len,
4918 .queue_num = priv->reta_idx_n,
4919 .key = priv->rss_conf.rss_key,
4922 struct rte_flow_action actions[] = {
4924 .type = RTE_FLOW_ACTION_TYPE_RSS,
4925 .conf = &action_rss,
4928 .type = RTE_FLOW_ACTION_TYPE_END,
4932 struct rte_flow_error error;
4935 if (!priv->reta_idx_n || !priv->rxqs_n) {
4938 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
4939 action_rss.types = 0;
4940 for (i = 0; i != priv->reta_idx_n; ++i)
4941 queue[i] = (*priv->reta_idx)[i];
4942 flow_idx = flow_list_create(dev, &priv->ctrl_flows,
4943 &attr, items, actions, false, &error);
4950 * Enable a flow control configured from the control plane.
4953 * Pointer to Ethernet device.
4955 * An Ethernet flow spec to apply.
4957 * An Ethernet flow mask to apply.
4960 * 0 on success, a negative errno value otherwise and rte_errno is set.
4963 mlx5_ctrl_flow(struct rte_eth_dev *dev,
4964 struct rte_flow_item_eth *eth_spec,
4965 struct rte_flow_item_eth *eth_mask)
4967 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
4973 * @see rte_flow_destroy()
4977 mlx5_flow_destroy(struct rte_eth_dev *dev,
4978 struct rte_flow *flow,
4979 struct rte_flow_error *error __rte_unused)
4981 struct mlx5_priv *priv = dev->data->dev_private;
4983 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
4988 * Destroy all flows.
4990 * @see rte_flow_flush()
4994 mlx5_flow_flush(struct rte_eth_dev *dev,
4995 struct rte_flow_error *error __rte_unused)
4997 struct mlx5_priv *priv = dev->data->dev_private;
4999 mlx5_flow_list_flush(dev, &priv->flows, false);
5006 * @see rte_flow_isolate()
5010 mlx5_flow_isolate(struct rte_eth_dev *dev,
5012 struct rte_flow_error *error)
5014 struct mlx5_priv *priv = dev->data->dev_private;
5016 if (dev->data->dev_started) {
5017 rte_flow_error_set(error, EBUSY,
5018 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5020 "port must be stopped first");
5023 priv->isolated = !!enable;
5025 dev->dev_ops = &mlx5_dev_ops_isolate;
5027 dev->dev_ops = &mlx5_dev_ops;
5034 * @see rte_flow_query()
5038 flow_drv_query(struct rte_eth_dev *dev,
5040 const struct rte_flow_action *actions,
5042 struct rte_flow_error *error)
5044 struct mlx5_priv *priv = dev->data->dev_private;
5045 const struct mlx5_flow_driver_ops *fops;
5046 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
5047 [MLX5_IPOOL_RTE_FLOW],
5049 enum mlx5_flow_drv_type ftype;
5052 return rte_flow_error_set(error, ENOENT,
5053 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5055 "invalid flow handle");
5057 ftype = flow->drv_type;
5058 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
5059 fops = flow_get_drv_ops(ftype);
5061 return fops->query(dev, flow, actions, data, error);
5067 * @see rte_flow_query()
5071 mlx5_flow_query(struct rte_eth_dev *dev,
5072 struct rte_flow *flow,
5073 const struct rte_flow_action *actions,
5075 struct rte_flow_error *error)
5079 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
5087 * Convert a flow director filter to a generic flow.
5090 * Pointer to Ethernet device.
5091 * @param fdir_filter
5092 * Flow director filter to add.
5094 * Generic flow parameters structure.
5097 * 0 on success, a negative errno value otherwise and rte_errno is set.
5100 flow_fdir_filter_convert(struct rte_eth_dev *dev,
5101 const struct rte_eth_fdir_filter *fdir_filter,
5102 struct mlx5_fdir *attributes)
5104 struct mlx5_priv *priv = dev->data->dev_private;
5105 const struct rte_eth_fdir_input *input = &fdir_filter->input;
5106 const struct rte_eth_fdir_masks *mask =
5107 &dev->data->dev_conf.fdir_conf.mask;
5109 /* Validate queue number. */
5110 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
5111 DRV_LOG(ERR, "port %u invalid queue number %d",
5112 dev->data->port_id, fdir_filter->action.rx_queue);
5116 attributes->attr.ingress = 1;
5117 attributes->items[0] = (struct rte_flow_item) {
5118 .type = RTE_FLOW_ITEM_TYPE_ETH,
5119 .spec = &attributes->l2,
5120 .mask = &attributes->l2_mask,
5122 switch (fdir_filter->action.behavior) {
5123 case RTE_ETH_FDIR_ACCEPT:
5124 attributes->actions[0] = (struct rte_flow_action){
5125 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
5126 .conf = &attributes->queue,
5129 case RTE_ETH_FDIR_REJECT:
5130 attributes->actions[0] = (struct rte_flow_action){
5131 .type = RTE_FLOW_ACTION_TYPE_DROP,
5135 DRV_LOG(ERR, "port %u invalid behavior %d",
5137 fdir_filter->action.behavior);
5138 rte_errno = ENOTSUP;
5141 attributes->queue.index = fdir_filter->action.rx_queue;
5143 switch (fdir_filter->input.flow_type) {
5144 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
5145 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
5146 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
5147 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){
5148 .src_addr = input->flow.ip4_flow.src_ip,
5149 .dst_addr = input->flow.ip4_flow.dst_ip,
5150 .time_to_live = input->flow.ip4_flow.ttl,
5151 .type_of_service = input->flow.ip4_flow.tos,
5153 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){
5154 .src_addr = mask->ipv4_mask.src_ip,
5155 .dst_addr = mask->ipv4_mask.dst_ip,
5156 .time_to_live = mask->ipv4_mask.ttl,
5157 .type_of_service = mask->ipv4_mask.tos,
5158 .next_proto_id = mask->ipv4_mask.proto,
5160 attributes->items[1] = (struct rte_flow_item){
5161 .type = RTE_FLOW_ITEM_TYPE_IPV4,
5162 .spec = &attributes->l3,
5163 .mask = &attributes->l3_mask,
5166 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
5167 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
5168 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
5169 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){
5170 .hop_limits = input->flow.ipv6_flow.hop_limits,
5171 .proto = input->flow.ipv6_flow.proto,
5174 memcpy(attributes->l3.ipv6.hdr.src_addr,
5175 input->flow.ipv6_flow.src_ip,
5176 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
5177 memcpy(attributes->l3.ipv6.hdr.dst_addr,
5178 input->flow.ipv6_flow.dst_ip,
5179 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
5180 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
5181 mask->ipv6_mask.src_ip,
5182 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
5183 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
5184 mask->ipv6_mask.dst_ip,
5185 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
5186 attributes->items[1] = (struct rte_flow_item){
5187 .type = RTE_FLOW_ITEM_TYPE_IPV6,
5188 .spec = &attributes->l3,
5189 .mask = &attributes->l3_mask,
5193 DRV_LOG(ERR, "port %u invalid flow type%d",
5194 dev->data->port_id, fdir_filter->input.flow_type);
5195 rte_errno = ENOTSUP;
5199 switch (fdir_filter->input.flow_type) {
5200 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
5201 attributes->l4.udp.hdr = (struct rte_udp_hdr){
5202 .src_port = input->flow.udp4_flow.src_port,
5203 .dst_port = input->flow.udp4_flow.dst_port,
5205 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
5206 .src_port = mask->src_port_mask,
5207 .dst_port = mask->dst_port_mask,
5209 attributes->items[2] = (struct rte_flow_item){
5210 .type = RTE_FLOW_ITEM_TYPE_UDP,
5211 .spec = &attributes->l4,
5212 .mask = &attributes->l4_mask,
5215 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
5216 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
5217 .src_port = input->flow.tcp4_flow.src_port,
5218 .dst_port = input->flow.tcp4_flow.dst_port,
5220 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
5221 .src_port = mask->src_port_mask,
5222 .dst_port = mask->dst_port_mask,
5224 attributes->items[2] = (struct rte_flow_item){
5225 .type = RTE_FLOW_ITEM_TYPE_TCP,
5226 .spec = &attributes->l4,
5227 .mask = &attributes->l4_mask,
5230 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
5231 attributes->l4.udp.hdr = (struct rte_udp_hdr){
5232 .src_port = input->flow.udp6_flow.src_port,
5233 .dst_port = input->flow.udp6_flow.dst_port,
5235 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){
5236 .src_port = mask->src_port_mask,
5237 .dst_port = mask->dst_port_mask,
5239 attributes->items[2] = (struct rte_flow_item){
5240 .type = RTE_FLOW_ITEM_TYPE_UDP,
5241 .spec = &attributes->l4,
5242 .mask = &attributes->l4_mask,
5245 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
5246 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){
5247 .src_port = input->flow.tcp6_flow.src_port,
5248 .dst_port = input->flow.tcp6_flow.dst_port,
5250 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){
5251 .src_port = mask->src_port_mask,
5252 .dst_port = mask->dst_port_mask,
5254 attributes->items[2] = (struct rte_flow_item){
5255 .type = RTE_FLOW_ITEM_TYPE_TCP,
5256 .spec = &attributes->l4,
5257 .mask = &attributes->l4_mask,
5260 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
5261 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
5264 DRV_LOG(ERR, "port %u invalid flow type%d",
5265 dev->data->port_id, fdir_filter->input.flow_type);
5266 rte_errno = ENOTSUP;
5272 #define FLOW_FDIR_CMP(f1, f2, fld) \
5273 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld))
5276 * Compare two FDIR flows. If items and actions are identical, the two flows are
5280 * Pointer to Ethernet device.
5282 * FDIR flow to compare.
5284 * FDIR flow to compare.
5287 * Zero on match, 1 otherwise.
5290 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2)
5292 if (FLOW_FDIR_CMP(f1, f2, attr) ||
5293 FLOW_FDIR_CMP(f1, f2, l2) ||
5294 FLOW_FDIR_CMP(f1, f2, l2_mask) ||
5295 FLOW_FDIR_CMP(f1, f2, l3) ||
5296 FLOW_FDIR_CMP(f1, f2, l3_mask) ||
5297 FLOW_FDIR_CMP(f1, f2, l4) ||
5298 FLOW_FDIR_CMP(f1, f2, l4_mask) ||
5299 FLOW_FDIR_CMP(f1, f2, actions[0].type))
5301 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
5302 FLOW_FDIR_CMP(f1, f2, queue))
5308 * Search device flow list to find out a matched FDIR flow.
5311 * Pointer to Ethernet device.
5313 * FDIR flow to lookup.
5316 * Index of flow if found, 0 otherwise.
5319 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow)
5321 struct mlx5_priv *priv = dev->data->dev_private;
5322 uint32_t flow_idx = 0;
5323 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5325 MLX5_ASSERT(fdir_flow);
5326 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5327 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) {
5328 DRV_LOG(DEBUG, "port %u found FDIR flow %u",
5329 dev->data->port_id, flow_idx);
5330 flow_idx = priv_fdir_flow->rix_flow;
5338 * Add new flow director filter and store it in list.
5341 * Pointer to Ethernet device.
5342 * @param fdir_filter
5343 * Flow director filter to add.
5346 * 0 on success, a negative errno value otherwise and rte_errno is set.
5349 flow_fdir_filter_add(struct rte_eth_dev *dev,
5350 const struct rte_eth_fdir_filter *fdir_filter)
5352 struct mlx5_priv *priv = dev->data->dev_private;
5353 struct mlx5_fdir *fdir_flow;
5354 struct rte_flow *flow;
5355 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5359 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0);
5364 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow);
5367 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow);
5372 priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow),
5374 if (!priv_fdir_flow) {
5378 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr,
5379 fdir_flow->items, fdir_flow->actions, true,
5381 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
5385 priv_fdir_flow->fdir = fdir_flow;
5386 priv_fdir_flow->rix_flow = flow_idx;
5387 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next);
5388 DRV_LOG(DEBUG, "port %u created FDIR flow %p",
5389 dev->data->port_id, (void *)flow);
5392 rte_free(priv_fdir_flow);
5393 rte_free(fdir_flow);
5398 * Delete specific filter.
5401 * Pointer to Ethernet device.
5402 * @param fdir_filter
5403 * Filter to be deleted.
5406 * 0 on success, a negative errno value otherwise and rte_errno is set.
5409 flow_fdir_filter_delete(struct rte_eth_dev *dev,
5410 const struct rte_eth_fdir_filter *fdir_filter)
5412 struct mlx5_priv *priv = dev->data->dev_private;
5414 struct mlx5_fdir fdir_flow = {
5417 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5420 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow);
5423 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
5424 /* Find the fdir in priv list */
5425 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow))
5428 if (!priv_fdir_flow)
5430 LIST_REMOVE(priv_fdir_flow, next);
5431 flow_idx = priv_fdir_flow->rix_flow;
5432 flow_list_destroy(dev, &priv->flows, flow_idx);
5433 rte_free(priv_fdir_flow->fdir);
5434 rte_free(priv_fdir_flow);
5435 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u",
5436 dev->data->port_id, flow_idx);
5441 * Update queue for specific filter.
5444 * Pointer to Ethernet device.
5445 * @param fdir_filter
5446 * Filter to be updated.
5449 * 0 on success, a negative errno value otherwise and rte_errno is set.
5452 flow_fdir_filter_update(struct rte_eth_dev *dev,
5453 const struct rte_eth_fdir_filter *fdir_filter)
5457 ret = flow_fdir_filter_delete(dev, fdir_filter);
5460 return flow_fdir_filter_add(dev, fdir_filter);
5464 * Flush all filters.
5467 * Pointer to Ethernet device.
5470 flow_fdir_filter_flush(struct rte_eth_dev *dev)
5472 struct mlx5_priv *priv = dev->data->dev_private;
5473 struct mlx5_fdir_flow *priv_fdir_flow = NULL;
5475 while (!LIST_EMPTY(&priv->fdir_flows)) {
5476 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows);
5477 LIST_REMOVE(priv_fdir_flow, next);
5478 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow);
5479 rte_free(priv_fdir_flow->fdir);
5480 rte_free(priv_fdir_flow);
5485 * Get flow director information.
5488 * Pointer to Ethernet device.
5489 * @param[out] fdir_info
5490 * Resulting flow director information.
5493 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
5495 struct rte_eth_fdir_masks *mask =
5496 &dev->data->dev_conf.fdir_conf.mask;
5498 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
5499 fdir_info->guarant_spc = 0;
5500 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
5501 fdir_info->max_flexpayload = 0;
5502 fdir_info->flow_types_mask[0] = 0;
5503 fdir_info->flex_payload_unit = 0;
5504 fdir_info->max_flex_payload_segment_num = 0;
5505 fdir_info->flex_payload_limit = 0;
5506 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
5510 * Deal with flow director operations.
5513 * Pointer to Ethernet device.
5515 * Operation to perform.
5517 * Pointer to operation-specific structure.
5520 * 0 on success, a negative errno value otherwise and rte_errno is set.
5523 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
5526 enum rte_fdir_mode fdir_mode =
5527 dev->data->dev_conf.fdir_conf.mode;
5529 if (filter_op == RTE_ETH_FILTER_NOP)
5531 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
5532 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
5533 DRV_LOG(ERR, "port %u flow director mode %d not supported",
5534 dev->data->port_id, fdir_mode);
5538 switch (filter_op) {
5539 case RTE_ETH_FILTER_ADD:
5540 return flow_fdir_filter_add(dev, arg);
5541 case RTE_ETH_FILTER_UPDATE:
5542 return flow_fdir_filter_update(dev, arg);
5543 case RTE_ETH_FILTER_DELETE:
5544 return flow_fdir_filter_delete(dev, arg);
5545 case RTE_ETH_FILTER_FLUSH:
5546 flow_fdir_filter_flush(dev);
5548 case RTE_ETH_FILTER_INFO:
5549 flow_fdir_info_get(dev, arg);
5552 DRV_LOG(DEBUG, "port %u unknown operation %u",
5553 dev->data->port_id, filter_op);
5561 * Manage filter operations.
5564 * Pointer to Ethernet device structure.
5565 * @param filter_type
5568 * Operation to perform.
5570 * Pointer to operation-specific structure.
5573 * 0 on success, a negative errno value otherwise and rte_errno is set.
5576 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
5577 enum rte_filter_type filter_type,
5578 enum rte_filter_op filter_op,
5581 switch (filter_type) {
5582 case RTE_ETH_FILTER_GENERIC:
5583 if (filter_op != RTE_ETH_FILTER_GET) {
5587 *(const void **)arg = &mlx5_flow_ops;
5589 case RTE_ETH_FILTER_FDIR:
5590 return flow_fdir_ctrl_func(dev, filter_op, arg);
5592 DRV_LOG(ERR, "port %u filter type (%d) not supported",
5593 dev->data->port_id, filter_type);
5594 rte_errno = ENOTSUP;
5601 * Create the needed meter and suffix tables.
5604 * Pointer to Ethernet device.
5606 * Pointer to the flow meter.
5609 * Pointer to table set on success, NULL otherwise.
5611 struct mlx5_meter_domains_infos *
5612 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
5613 const struct mlx5_flow_meter *fm)
5615 const struct mlx5_flow_driver_ops *fops;
5617 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5618 return fops->create_mtr_tbls(dev, fm);
5622 * Destroy the meter table set.
5625 * Pointer to Ethernet device.
5627 * Pointer to the meter table set.
5633 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
5634 struct mlx5_meter_domains_infos *tbls)
5636 const struct mlx5_flow_driver_ops *fops;
5638 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5639 return fops->destroy_mtr_tbls(dev, tbls);
5643 * Create policer rules.
5646 * Pointer to Ethernet device.
5648 * Pointer to flow meter structure.
5650 * Pointer to flow attributes.
5653 * 0 on success, -1 otherwise.
5656 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
5657 struct mlx5_flow_meter *fm,
5658 const struct rte_flow_attr *attr)
5660 const struct mlx5_flow_driver_ops *fops;
5662 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5663 return fops->create_policer_rules(dev, fm, attr);
5667 * Destroy policer rules.
5670 * Pointer to flow meter structure.
5672 * Pointer to flow attributes.
5675 * 0 on success, -1 otherwise.
5678 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
5679 struct mlx5_flow_meter *fm,
5680 const struct rte_flow_attr *attr)
5682 const struct mlx5_flow_driver_ops *fops;
5684 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5685 return fops->destroy_policer_rules(dev, fm, attr);
5689 * Allocate a counter.
5692 * Pointer to Ethernet device structure.
5695 * Index to allocated counter on success, 0 otherwise.
5698 mlx5_counter_alloc(struct rte_eth_dev *dev)
5700 const struct mlx5_flow_driver_ops *fops;
5701 struct rte_flow_attr attr = { .transfer = 0 };
5703 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5704 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5705 return fops->counter_alloc(dev);
5708 "port %u counter allocate is not supported.",
5709 dev->data->port_id);
5717 * Pointer to Ethernet device structure.
5719 * Index to counter to be free.
5722 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
5724 const struct mlx5_flow_driver_ops *fops;
5725 struct rte_flow_attr attr = { .transfer = 0 };
5727 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5728 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5729 fops->counter_free(dev, cnt);
5733 "port %u counter free is not supported.",
5734 dev->data->port_id);
5738 * Query counter statistics.
5741 * Pointer to Ethernet device structure.
5743 * Index to counter to query.
5745 * Set to clear counter statistics.
5747 * The counter hits packets number to save.
5749 * The counter hits bytes number to save.
5752 * 0 on success, a negative errno value otherwise.
5755 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
5756 bool clear, uint64_t *pkts, uint64_t *bytes)
5758 const struct mlx5_flow_driver_ops *fops;
5759 struct rte_flow_attr attr = { .transfer = 0 };
5761 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
5762 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
5763 return fops->counter_query(dev, cnt, clear, pkts, bytes);
5766 "port %u counter query is not supported.",
5767 dev->data->port_id);
5771 #define MLX5_POOL_QUERY_FREQ_US 1000000
5774 * Get number of all validate pools.
5777 * Pointer to mlx5_ibv_shared object.
5780 * The number of all validate pools.
5783 mlx5_get_all_valid_pool_count(struct mlx5_ibv_shared *sh)
5786 uint32_t pools_n = 0;
5787 struct mlx5_pools_container *cont;
5789 for (age = 0; age < RTE_DIM(sh->cmng.ccont[0]); ++age) {
5790 for (i = 0; i < 2 ; ++i) {
5791 cont = MLX5_CNT_CONTAINER(sh, i, 0, age);
5792 pools_n += rte_atomic16_read(&cont->n_valid);
5799 * Set the periodic procedure for triggering asynchronous batch queries for all
5800 * the counter pools.
5803 * Pointer to mlx5_ibv_shared object.
5806 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh)
5808 uint32_t pools_n, us;
5810 pools_n = mlx5_get_all_valid_pool_count(sh);
5811 us = MLX5_POOL_QUERY_FREQ_US / pools_n;
5812 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
5813 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
5814 sh->cmng.query_thread_on = 0;
5815 DRV_LOG(ERR, "Cannot reinitialize query alarm");
5817 sh->cmng.query_thread_on = 1;
5822 * The periodic procedure for triggering asynchronous batch queries for all the
5823 * counter pools. This function is probably called by the host thread.
5826 * The parameter for the alarm process.
5829 mlx5_flow_query_alarm(void *arg)
5831 struct mlx5_ibv_shared *sh = arg;
5832 struct mlx5_devx_obj *dcs;
5835 uint8_t batch = sh->cmng.batch;
5836 uint8_t age = sh->cmng.age;
5837 uint16_t pool_index = sh->cmng.pool_index;
5838 struct mlx5_pools_container *cont;
5839 struct mlx5_pools_container *mcont;
5840 struct mlx5_flow_counter_pool *pool;
5842 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
5845 cont = MLX5_CNT_CONTAINER(sh, batch, 1, age);
5846 mcont = MLX5_CNT_CONTAINER(sh, batch, 0, age);
5847 /* Check if resize was done and need to flip a container. */
5848 if (cont != mcont) {
5850 /* Clean the old container. */
5851 rte_free(cont->pools);
5852 memset(cont, 0, sizeof(*cont));
5855 /* Flip the host container. */
5856 sh->cmng.mhi[batch][age] ^= (uint8_t)2;
5860 /* 2 empty containers case is unexpected. */
5861 if (unlikely(batch != sh->cmng.batch) &&
5862 unlikely(age != sh->cmng.age)) {
5867 if (batch == 0 && pool_index == 0) {
5869 sh->cmng.batch = batch;
5872 goto next_container;
5874 pool = cont->pools[pool_index];
5876 /* There is a pool query in progress. */
5879 LIST_FIRST(&sh->cmng.free_stat_raws);
5881 /* No free counter statistics raw memory. */
5883 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read
5885 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL;
5887 * Identify the counters released between query trigger and query
5888 * handle more effiecntly. The counter released in this gap period
5889 * should wait for a new round of query as the new arrived packets
5890 * will not be taken into account.
5892 rte_atomic64_add(&pool->start_query_gen, 1);
5893 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL -
5895 pool->raw_hw->mem_mng->dm->id,
5897 (pool->raw_hw->data + offset),
5899 (uint64_t)(uintptr_t)pool);
5901 rte_atomic64_sub(&pool->start_query_gen, 1);
5902 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
5903 " %d", pool->min_dcs->id);
5904 pool->raw_hw = NULL;
5907 pool->raw_hw->min_dcs_id = dcs->id;
5908 LIST_REMOVE(pool->raw_hw, next);
5909 sh->cmng.pending_queries++;
5911 if (pool_index >= rte_atomic16_read(&cont->n_valid)) {
5914 if (batch == 0 && pool_index == 0)
5918 sh->cmng.batch = batch;
5919 sh->cmng.pool_index = pool_index;
5921 mlx5_set_query_alarm(sh);
5925 * Check and callback event for new aged flow in the counter pool
5928 * Pointer to mlx5_ibv_shared object.
5930 * Pointer to Current counter pool.
5933 mlx5_flow_aging_check(struct mlx5_ibv_shared *sh,
5934 struct mlx5_flow_counter_pool *pool)
5936 struct mlx5_priv *priv;
5937 struct mlx5_flow_counter *cnt;
5938 struct mlx5_age_info *age_info;
5939 struct mlx5_age_param *age_param;
5940 struct mlx5_counter_stats_raw *cur = pool->raw_hw;
5941 struct mlx5_counter_stats_raw *prev = pool->raw;
5942 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10);
5945 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
5946 cnt = MLX5_POOL_GET_CNT(pool, i);
5947 age_param = MLX5_CNT_TO_AGE(cnt);
5948 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE)
5950 if (cur->data[i].hits != prev->data[i].hits) {
5951 age_param->expire = curr + age_param->timeout;
5954 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2))
5957 * Hold the lock first, or if between the
5958 * state AGE_TMOUT and tailq operation the
5959 * release happened, the release procedure
5960 * may delete a non-existent tailq node.
5962 priv = rte_eth_devices[age_param->port_id].data->dev_private;
5963 age_info = GET_PORT_AGE_INFO(priv);
5964 rte_spinlock_lock(&age_info->aged_sl);
5965 /* If the cpmset fails, release happens. */
5966 if (rte_atomic16_cmpset((volatile uint16_t *)
5971 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
5972 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
5974 rte_spinlock_unlock(&age_info->aged_sl);
5976 for (i = 0; i < sh->max_port; i++) {
5977 age_info = &sh->port[i].age_info;
5978 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW))
5980 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER))
5981 _rte_eth_dev_callback_process
5982 (&rte_eth_devices[sh->port[i].devx_ih_port_id],
5983 RTE_ETH_EVENT_FLOW_AGED, NULL);
5984 age_info->flags = 0;
5989 * Handler for the HW respond about ready values from an asynchronous batch
5990 * query. This function is probably called by the host thread.
5993 * The pointer to the shared IB device context.
5994 * @param[in] async_id
5995 * The Devx async ID.
5997 * The status of the completion.
6000 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh,
6001 uint64_t async_id, int status)
6003 struct mlx5_flow_counter_pool *pool =
6004 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
6005 struct mlx5_counter_stats_raw *raw_to_free;
6007 if (unlikely(status)) {
6008 rte_atomic64_sub(&pool->start_query_gen, 1);
6009 raw_to_free = pool->raw_hw;
6011 raw_to_free = pool->raw;
6012 if (IS_AGE_POOL(pool))
6013 mlx5_flow_aging_check(sh, pool);
6014 rte_spinlock_lock(&pool->sl);
6015 pool->raw = pool->raw_hw;
6016 rte_spinlock_unlock(&pool->sl);
6017 MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 ==
6018 rte_atomic64_read(&pool->start_query_gen));
6019 rte_atomic64_set(&pool->end_query_gen,
6020 rte_atomic64_read(&pool->start_query_gen));
6021 /* Be sure the new raw counters data is updated in memory. */
6024 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
6025 pool->raw_hw = NULL;
6026 sh->cmng.pending_queries--;
6030 * Translate the rte_flow group index to HW table value.
6032 * @param[in] attributes
6033 * Pointer to flow attributes
6034 * @param[in] external
6035 * Value is part of flow rule created by request external to PMD.
6037 * rte_flow group index value.
6038 * @param[out] fdb_def_rule
6039 * Whether fdb jump to table 1 is configured.
6043 * Pointer to error structure.
6046 * 0 on success, a negative errno value otherwise and rte_errno is set.
6049 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external,
6050 uint32_t group, bool fdb_def_rule, uint32_t *table,
6051 struct rte_flow_error *error)
6053 if (attributes->transfer && external && fdb_def_rule) {
6054 if (group == UINT32_MAX)
6055 return rte_flow_error_set
6057 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6059 "group index not supported");
6068 * Discover availability of metadata reg_c's.
6070 * Iteratively use test flows to check availability.
6073 * Pointer to the Ethernet device structure.
6076 * 0 on success, a negative errno value otherwise and rte_errno is set.
6079 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
6081 struct mlx5_priv *priv = dev->data->dev_private;
6082 struct mlx5_dev_config *config = &priv->config;
6083 enum modify_reg idx;
6086 /* reg_c[0] and reg_c[1] are reserved. */
6087 config->flow_mreg_c[n++] = REG_C_0;
6088 config->flow_mreg_c[n++] = REG_C_1;
6089 /* Discover availability of other reg_c's. */
6090 for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
6091 struct rte_flow_attr attr = {
6092 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6093 .priority = MLX5_FLOW_PRIO_RSVD,
6096 struct rte_flow_item items[] = {
6098 .type = RTE_FLOW_ITEM_TYPE_END,
6101 struct rte_flow_action actions[] = {
6103 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6104 .conf = &(struct mlx5_flow_action_copy_mreg){
6110 .type = RTE_FLOW_ACTION_TYPE_JUMP,
6111 .conf = &(struct rte_flow_action_jump){
6112 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6116 .type = RTE_FLOW_ACTION_TYPE_END,
6120 struct rte_flow *flow;
6121 struct rte_flow_error error;
6123 if (!config->dv_flow_en)
6125 /* Create internal flow, validation skips copy action. */
6126 flow_idx = flow_list_create(dev, NULL, &attr, items,
6127 actions, false, &error);
6128 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
6132 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL))
6133 config->flow_mreg_c[n++] = idx;
6134 flow_list_destroy(dev, NULL, flow_idx);
6136 for (; n < MLX5_MREG_C_NUM; ++n)
6137 config->flow_mreg_c[n] = REG_NONE;
6142 * Dump flow raw hw data to file
6145 * The pointer to Ethernet device.
6147 * A pointer to a file for output.
6149 * Perform verbose error reporting if not NULL. PMDs initialize this
6150 * structure in case of error only.
6152 * 0 on success, a nagative value otherwise.
6155 mlx5_flow_dev_dump(struct rte_eth_dev *dev,
6157 struct rte_flow_error *error __rte_unused)
6159 struct mlx5_priv *priv = dev->data->dev_private;
6160 struct mlx5_ibv_shared *sh = priv->sh;
6162 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain,
6163 sh->tx_domain, file);
6167 * Get aged-out flows.
6170 * Pointer to the Ethernet device structure.
6171 * @param[in] context
6172 * The address of an array of pointers to the aged-out flows contexts.
6173 * @param[in] nb_countexts
6174 * The length of context array pointers.
6176 * Perform verbose error reporting if not NULL. Initialized in case of
6180 * how many contexts get in success, otherwise negative errno value.
6181 * if nb_contexts is 0, return the amount of all aged contexts.
6182 * if nb_contexts is not 0 , return the amount of aged flows reported
6183 * in the context array.
6186 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
6187 uint32_t nb_contexts, struct rte_flow_error *error)
6189 const struct mlx5_flow_driver_ops *fops;
6190 struct rte_flow_attr attr = { .transfer = 0 };
6192 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
6193 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
6194 return fops->get_aged_flows(dev, contexts, nb_contexts,
6198 "port %u get aged flows is not supported.",
6199 dev->data->port_id);