1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
17 #include <infiniband/mlx5dv.h>
19 #pragma GCC diagnostic error "-Wpedantic"
23 #include <rte_mempool.h>
24 #include <rte_prefetch.h>
27 #include "mlx5_utils.h"
28 #include "mlx5_rxtx.h"
29 #include "mlx5_rxtx_vec.h"
30 #include "mlx5_autoconf.h"
31 #include "mlx5_defs.h"
34 #if defined RTE_ARCH_X86_64
35 #include "mlx5_rxtx_vec_sse.h"
36 #elif defined RTE_ARCH_ARM64
37 #include "mlx5_rxtx_vec_neon.h"
39 #error "This should not be compiled if SIMD instructions are not supported."
43 * Count the number of packets having same ol_flags and calculate cs_flags.
46 * Pointer to array of packets.
50 * Pointer of flags to be returned.
53 * Number of packets having same ol_flags.
55 static inline unsigned int
56 txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags)
59 const uint64_t ol_mask =
60 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
61 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
62 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
66 /* Count the number of packets having same ol_flags. */
67 for (pos = 1; pos < pkts_n; ++pos)
68 if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
70 *cs_flags = txq_ol_cksum_to_cs(pkts[0]);
75 * DPDK callback for vectorized TX.
78 * Generic pointer to TX queue structure.
80 * Packets to transmit.
82 * Number of packets in array.
85 * Number of packets successfully transmitted (<= pkts_n).
88 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
91 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
94 while (pkts_n > nb_tx) {
98 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
99 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
108 * DPDK callback for vectorized TX with multi-seg packets and offload.
111 * Generic pointer to TX queue structure.
113 * Packets to transmit.
115 * Number of packets in array.
118 * Number of packets successfully transmitted (<= pkts_n).
121 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
123 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
126 while (pkts_n > nb_tx) {
127 uint8_t cs_flags = 0;
131 /* Transmit multi-seg packets in the head of pkts list. */
132 if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) &&
133 NB_SEGS(pkts[nb_tx]) > 1)
134 nb_tx += txq_scatter_v(txq,
137 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
138 if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
139 n = txq_count_contig_single_seg(&pkts[nb_tx], n);
140 if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP)
141 n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags);
142 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
151 * Skip error packets.
154 * Pointer to RX queue structure.
156 * Array to store received packets.
158 * Maximum number of packets in array.
161 * Number of packets successfully received (<= pkts_n).
164 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
169 #ifdef MLX5_PMD_SOFT_COUNTERS
170 uint32_t err_bytes = 0;
173 for (i = 0; i < pkts_n; ++i) {
174 struct rte_mbuf *pkt = pkts[i];
176 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) {
177 #ifdef MLX5_PMD_SOFT_COUNTERS
178 err_bytes += PKT_LEN(pkt);
180 rte_pktmbuf_free_seg(pkt);
185 rxq->stats.idropped += (pkts_n - n);
186 #ifdef MLX5_PMD_SOFT_COUNTERS
187 /* Correct counters of errored completions. */
188 rxq->stats.ipackets -= (pkts_n - n);
189 rxq->stats.ibytes -= err_bytes;
195 * DPDK callback for vectorized RX.
198 * Generic pointer to RX queue structure.
200 * Array to store received packets.
202 * Maximum number of packets in array.
205 * Number of packets successfully received (<= pkts_n).
208 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
210 struct mlx5_rxq_data *rxq = dpdk_rxq;
214 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
216 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
221 * Check Tx queue flags are set for raw vectorized Tx.
224 * Pointer to Ethernet device.
227 * 1 if supported, negative errno value if not.
229 int __attribute__((cold))
230 mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev)
232 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
234 /* Doesn't support any offload. */
241 * Check a device can support vectorized TX.
244 * Pointer to Ethernet device.
247 * 1 if supported, negative errno value if not.
249 int __attribute__((cold))
250 mlx5_check_vec_tx_support(struct rte_eth_dev *dev)
252 struct priv *priv = dev->data->dev_private;
253 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
255 if (!priv->config.tx_vec_en ||
256 priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
257 priv->config.mps != MLX5_MPW_ENHANCED ||
258 offloads & ~MLX5_VEC_TX_OFFLOAD_CAP)
264 * Check a RX queue can support vectorized RX.
267 * Pointer to RX queue.
270 * 1 if supported, negative errno value if not.
272 int __attribute__((cold))
273 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
275 struct mlx5_rxq_ctrl *ctrl =
276 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
278 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
280 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
286 * Check a device can support vectorized RX.
289 * Pointer to Ethernet device.
292 * 1 if supported, negative errno value if not.
294 int __attribute__((cold))
295 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
297 struct priv *priv = dev->data->dev_private;
300 if (!priv->config.rx_vec_en)
302 if (mlx5_mprq_enabled(dev))
304 /* All the configured queues should support. */
305 for (i = 0; i < priv->rxqs_n; ++i) {
306 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
310 if (mlx5_rxq_check_vec_support(rxq) < 0)
313 if (i != priv->rxqs_n)