4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
42 #pragma GCC diagnostic ignored "-Wpedantic"
44 #include <infiniband/verbs.h>
45 #include <infiniband/mlx5dv.h>
47 #pragma GCC diagnostic error "-Wpedantic"
51 #include <rte_mempool.h>
52 #include <rte_prefetch.h>
55 #include "mlx5_utils.h"
56 #include "mlx5_rxtx.h"
57 #include "mlx5_rxtx_vec.h"
58 #include "mlx5_autoconf.h"
59 #include "mlx5_defs.h"
62 #if defined RTE_ARCH_X86_64
63 #include "mlx5_rxtx_vec_sse.h"
64 #elif defined RTE_ARCH_ARM64
65 #include "mlx5_rxtx_vec_neon.h"
67 #error "This should not be compiled if SIMD instructions are not supported."
71 * Count the number of continuous single segment packets.
74 * Pointer to array of packets.
79 * Number of continuous single segment packets.
81 static inline unsigned int
82 txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n)
88 /* Count the number of continuous single segment packets. */
89 for (pos = 0; pos < pkts_n; ++pos)
90 if (NB_SEGS(pkts[pos]) > 1)
96 * Count the number of packets having same ol_flags and calculate cs_flags.
99 * Pointer to TX queue structure.
101 * Pointer to array of packets.
105 * Pointer of flags to be returned.
108 * Number of packets having same ol_flags.
110 static inline unsigned int
111 txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
112 uint16_t pkts_n, uint8_t *cs_flags)
115 const uint64_t ol_mask =
116 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
117 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
118 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
122 /* Count the number of packets having same ol_flags. */
123 for (pos = 1; pos < pkts_n; ++pos)
124 if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
126 /* Should open another MPW session for the rest. */
127 if (pkts[0]->ol_flags &
128 (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
129 const uint64_t is_tunneled =
132 PKT_TX_TUNNEL_VXLAN);
134 if (is_tunneled && txq->tunnel_en) {
135 *cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
136 MLX5_ETH_WQE_L4_INNER_CSUM;
137 if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM)
138 *cs_flags |= MLX5_ETH_WQE_L3_CSUM;
140 *cs_flags = MLX5_ETH_WQE_L3_CSUM |
141 MLX5_ETH_WQE_L4_CSUM;
148 * DPDK callback for vectorized TX.
151 * Generic pointer to TX queue structure.
153 * Packets to transmit.
155 * Number of packets in array.
158 * Number of packets successfully transmitted (<= pkts_n).
161 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
164 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
167 while (pkts_n > nb_tx) {
171 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
172 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
181 * DPDK callback for vectorized TX with multi-seg packets and offload.
184 * Generic pointer to TX queue structure.
186 * Packets to transmit.
188 * Number of packets in array.
191 * Number of packets successfully transmitted (<= pkts_n).
194 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
196 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
199 while (pkts_n > nb_tx) {
200 uint8_t cs_flags = 0;
204 /* Transmit multi-seg packets in the head of pkts list. */
205 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) &&
206 NB_SEGS(pkts[nb_tx]) > 1)
207 nb_tx += txq_scatter_v(txq,
210 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
211 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
212 n = txq_check_multiseg(&pkts[nb_tx], n);
213 if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
214 n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
215 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
224 * Skip error packets.
227 * Pointer to RX queue structure.
229 * Array to store received packets.
231 * Maximum number of packets in array.
234 * Number of packets successfully received (<= pkts_n).
237 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
242 #ifdef MLX5_PMD_SOFT_COUNTERS
243 uint32_t err_bytes = 0;
246 for (i = 0; i < pkts_n; ++i) {
247 struct rte_mbuf *pkt = pkts[i];
249 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) {
250 #ifdef MLX5_PMD_SOFT_COUNTERS
251 err_bytes += PKT_LEN(pkt);
253 rte_pktmbuf_free_seg(pkt);
258 rxq->stats.idropped += (pkts_n - n);
259 #ifdef MLX5_PMD_SOFT_COUNTERS
260 /* Correct counters of errored completions. */
261 rxq->stats.ipackets -= (pkts_n - n);
262 rxq->stats.ibytes -= err_bytes;
264 rxq->pending_err = 0;
269 * DPDK callback for vectorized RX.
272 * Generic pointer to RX queue structure.
274 * Array to store received packets.
276 * Maximum number of packets in array.
279 * Number of packets successfully received (<= pkts_n).
282 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
284 struct mlx5_rxq_data *rxq = dpdk_rxq;
287 nb_rx = rxq_burst_v(rxq, pkts, pkts_n);
288 if (unlikely(rxq->pending_err))
289 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
294 * Check Tx queue flags are set for raw vectorized Tx.
297 * Pointer to private structure.
300 * 1 if supported, negative errno value if not.
302 int __attribute__((cold))
303 priv_check_raw_vec_tx_support(struct priv *priv)
307 /* All the configured queues should support. */
308 for (i = 0; i < priv->txqs_n; ++i) {
309 struct mlx5_txq_data *txq = (*priv->txqs)[i];
311 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) ||
312 !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
315 if (i != priv->txqs_n)
321 * Check a device can support vectorized TX.
324 * Pointer to private structure.
327 * 1 if supported, negative errno value if not.
329 int __attribute__((cold))
330 priv_check_vec_tx_support(struct priv *priv)
332 if (!priv->tx_vec_en ||
333 priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
334 priv->mps != MLX5_MPW_ENHANCED ||
341 * Check a RX queue can support vectorized RX.
344 * Pointer to RX queue.
347 * 1 if supported, negative errno value if not.
349 int __attribute__((cold))
350 rxq_check_vec_support(struct mlx5_rxq_data *rxq)
352 struct mlx5_rxq_ctrl *ctrl =
353 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
355 if (!ctrl->priv->rx_vec_en || rxq->sges_n != 0)
361 * Check a device can support vectorized RX.
364 * Pointer to private structure.
367 * 1 if supported, negative errno value if not.
369 int __attribute__((cold))
370 priv_check_vec_rx_support(struct priv *priv)
374 if (!priv->rx_vec_en)
376 /* All the configured queues should support. */
377 for (i = 0; i < priv->rxqs_n; ++i) {
378 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
382 if (rxq_check_vec_support(rxq) < 0)
385 if (i != priv->rxqs_n)