net/qede: fix odd number of queues usage in 100G mode
[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx_vec.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2017 6WIND S.A.
3  * Copyright 2017 Mellanox Technologies, Ltd
4  */
5
6 #include <assert.h>
7 #include <stdint.h>
8 #include <string.h>
9 #include <stdlib.h>
10
11 /* Verbs header. */
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
13 #ifdef PEDANTIC
14 #pragma GCC diagnostic ignored "-Wpedantic"
15 #endif
16 #include <infiniband/verbs.h>
17 #include <infiniband/mlx5dv.h>
18 #ifdef PEDANTIC
19 #pragma GCC diagnostic error "-Wpedantic"
20 #endif
21
22 #include <rte_mbuf.h>
23 #include <rte_mempool.h>
24 #include <rte_prefetch.h>
25
26 #include "mlx5.h"
27 #include "mlx5_utils.h"
28 #include "mlx5_rxtx.h"
29 #include "mlx5_rxtx_vec.h"
30 #include "mlx5_autoconf.h"
31 #include "mlx5_defs.h"
32 #include "mlx5_prm.h"
33
34 #if defined RTE_ARCH_X86_64
35 #include "mlx5_rxtx_vec_sse.h"
36 #elif defined RTE_ARCH_ARM64
37 #include "mlx5_rxtx_vec_neon.h"
38 #else
39 #error "This should not be compiled if SIMD instructions are not supported."
40 #endif
41
42 /**
43  * Skip error packets.
44  *
45  * @param rxq
46  *   Pointer to RX queue structure.
47  * @param[out] pkts
48  *   Array to store received packets.
49  * @param pkts_n
50  *   Maximum number of packets in array.
51  *
52  * @return
53  *   Number of packets successfully received (<= pkts_n).
54  */
55 static uint16_t
56 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
57                          uint16_t pkts_n)
58 {
59         uint16_t n = 0;
60         unsigned int i;
61 #ifdef MLX5_PMD_SOFT_COUNTERS
62         uint32_t err_bytes = 0;
63 #endif
64
65         for (i = 0; i < pkts_n; ++i) {
66                 struct rte_mbuf *pkt = pkts[i];
67
68                 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
69 #ifdef MLX5_PMD_SOFT_COUNTERS
70                         err_bytes += PKT_LEN(pkt);
71 #endif
72                         rte_pktmbuf_free_seg(pkt);
73                 } else {
74                         pkts[n++] = pkt;
75                 }
76         }
77         rxq->stats.idropped += (pkts_n - n);
78 #ifdef MLX5_PMD_SOFT_COUNTERS
79         /* Correct counters of errored completions. */
80         rxq->stats.ipackets -= (pkts_n - n);
81         rxq->stats.ibytes -= err_bytes;
82 #endif
83         mlx5_rx_err_handle(rxq, 1);
84         return n;
85 }
86
87 /**
88  * DPDK callback for vectorized RX.
89  *
90  * @param dpdk_rxq
91  *   Generic pointer to RX queue structure.
92  * @param[out] pkts
93  *   Array to store received packets.
94  * @param pkts_n
95  *   Maximum number of packets in array.
96  *
97  * @return
98  *   Number of packets successfully received (<= pkts_n).
99  */
100 uint16_t
101 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
102 {
103         struct mlx5_rxq_data *rxq = dpdk_rxq;
104         uint16_t nb_rx;
105         uint64_t err = 0;
106
107         nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
108         if (unlikely(err | rxq->err_state))
109                 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
110         return nb_rx;
111 }
112
113 /**
114  * Check a RX queue can support vectorized RX.
115  *
116  * @param rxq
117  *   Pointer to RX queue.
118  *
119  * @return
120  *   1 if supported, negative errno value if not.
121  */
122 int __attribute__((cold))
123 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
124 {
125         struct mlx5_rxq_ctrl *ctrl =
126                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
127
128         if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
129                 return -ENOTSUP;
130         if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
131                 return -ENOTSUP;
132         if (rxq->lro)
133                 return -ENOTSUP;
134         return 1;
135 }
136
137 /**
138  * Check a device can support vectorized RX.
139  *
140  * @param dev
141  *   Pointer to Ethernet device.
142  *
143  * @return
144  *   1 if supported, negative errno value if not.
145  */
146 int __attribute__((cold))
147 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
148 {
149         struct mlx5_priv *priv = dev->data->dev_private;
150         uint16_t i;
151
152         if (!priv->config.rx_vec_en)
153                 return -ENOTSUP;
154         if (mlx5_mprq_enabled(dev))
155                 return -ENOTSUP;
156         /* All the configured queues should support. */
157         for (i = 0; i < priv->rxqs_n; ++i) {
158                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
159
160                 if (!rxq)
161                         continue;
162                 if (mlx5_rxq_check_vec_support(rxq) < 0)
163                         break;
164         }
165         if (i != priv->rxqs_n)
166                 return -ENOTSUP;
167         return 1;
168 }