1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
17 #include <infiniband/mlx5dv.h>
19 #pragma GCC diagnostic error "-Wpedantic"
23 #include <rte_mempool.h>
24 #include <rte_prefetch.h>
28 #include "mlx5_defs.h"
30 #include "mlx5_utils.h"
31 #include "mlx5_rxtx.h"
32 #include "mlx5_rxtx_vec.h"
33 #include "mlx5_autoconf.h"
35 #if defined RTE_ARCH_X86_64
36 #include "mlx5_rxtx_vec_sse.h"
37 #elif defined RTE_ARCH_ARM64
38 #include "mlx5_rxtx_vec_neon.h"
39 #elif defined RTE_ARCH_PPC_64
40 #include "mlx5_rxtx_vec_altivec.h"
42 #error "This should not be compiled if SIMD instructions are not supported."
49 * Pointer to RX queue structure.
51 * Array to store received packets.
53 * Maximum number of packets in array.
56 * Number of packets successfully received (<= pkts_n).
59 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
64 #ifdef MLX5_PMD_SOFT_COUNTERS
65 uint32_t err_bytes = 0;
68 for (i = 0; i < pkts_n; ++i) {
69 struct rte_mbuf *pkt = pkts[i];
71 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
72 #ifdef MLX5_PMD_SOFT_COUNTERS
73 err_bytes += PKT_LEN(pkt);
75 rte_pktmbuf_free_seg(pkt);
80 rxq->stats.idropped += (pkts_n - n);
81 #ifdef MLX5_PMD_SOFT_COUNTERS
82 /* Correct counters of errored completions. */
83 rxq->stats.ipackets -= (pkts_n - n);
84 rxq->stats.ibytes -= err_bytes;
86 mlx5_rx_err_handle(rxq, 1);
91 * DPDK callback for vectorized RX.
94 * Generic pointer to RX queue structure.
96 * Array to store received packets.
98 * Maximum number of packets in array.
101 * Number of packets successfully received (<= pkts_n).
104 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
106 struct mlx5_rxq_data *rxq = dpdk_rxq;
110 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
111 if (unlikely(err | rxq->err_state))
112 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
117 * Check a RX queue can support vectorized RX.
120 * Pointer to RX queue.
123 * 1 if supported, negative errno value if not.
125 int __attribute__((cold))
126 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
128 struct mlx5_rxq_ctrl *ctrl =
129 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
131 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
133 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
141 * Check a device can support vectorized RX.
144 * Pointer to Ethernet device.
147 * 1 if supported, negative errno value if not.
149 int __attribute__((cold))
150 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
152 struct mlx5_priv *priv = dev->data->dev_private;
155 if (!priv->config.rx_vec_en)
157 if (mlx5_mprq_enabled(dev))
159 /* All the configured queues should support. */
160 for (i = 0; i < priv->rxqs_n; ++i) {
161 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
165 if (mlx5_rxq_check_vec_support(rxq) < 0)
168 if (i != priv->rxqs_n)