1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2015 6WIND S.A.
3 * Copyright 2015 Mellanox Technologies, Ltd
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
17 #include <infiniband/mlx5dv.h>
19 #pragma GCC diagnostic error "-Wpedantic"
23 #include <rte_mempool.h>
24 #include <rte_prefetch.h>
25 #include <rte_common.h>
26 #include <rte_branch_prediction.h>
27 #include <rte_ether.h>
28 #include <rte_cycles.h>
31 #include "mlx5_utils.h"
32 #include "mlx5_rxtx.h"
33 #include "mlx5_autoconf.h"
34 #include "mlx5_defs.h"
37 static __rte_always_inline uint32_t
38 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe);
40 static __rte_always_inline int
41 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
42 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe);
44 static __rte_always_inline uint32_t
45 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe);
47 static __rte_always_inline void
48 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
49 volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res);
51 static __rte_always_inline void
52 mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx);
54 uint32_t mlx5_ptype_table[] __rte_cache_aligned = {
55 [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */
58 uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned;
59 uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
62 * Build a table to translate Rx completion flags to packet type.
64 * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
67 mlx5_set_ptype_table(void)
70 uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table;
72 /* Last entry must not be overwritten, reserved for errored packet. */
73 for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i)
74 (*p)[i] = RTE_PTYPE_UNKNOWN;
76 * The index to the array should have:
77 * bit[1:0] = l3_hdr_type
78 * bit[4:2] = l4_hdr_type
81 * bit[7] = outer_l3_type
84 (*p)[0x00] = RTE_PTYPE_L2_ETHER;
86 (*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
88 (*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
91 (*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
93 (*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
96 (*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
98 (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
100 (*p)[0x0d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
102 (*p)[0x0e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
104 (*p)[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
106 (*p)[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
109 (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
111 (*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
113 /* Repeat with outer_l3_type being set. Just in case. */
114 (*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
115 RTE_PTYPE_L4_NONFRAG;
116 (*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
117 RTE_PTYPE_L4_NONFRAG;
118 (*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
120 (*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
122 (*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
124 (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
126 (*p)[0x8d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
128 (*p)[0x8e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
130 (*p)[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
132 (*p)[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
134 (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
136 (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
139 (*p)[0x40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
140 (*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
141 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
142 RTE_PTYPE_INNER_L4_NONFRAG;
143 (*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
144 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
145 RTE_PTYPE_INNER_L4_NONFRAG;
146 (*p)[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
147 (*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
148 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
149 RTE_PTYPE_INNER_L4_NONFRAG;
150 (*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
151 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
152 RTE_PTYPE_INNER_L4_NONFRAG;
153 /* Tunneled - Fragmented */
154 (*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
155 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
156 RTE_PTYPE_INNER_L4_FRAG;
157 (*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
158 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
159 RTE_PTYPE_INNER_L4_FRAG;
160 (*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
161 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
162 RTE_PTYPE_INNER_L4_FRAG;
163 (*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
164 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
165 RTE_PTYPE_INNER_L4_FRAG;
167 (*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
168 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
169 RTE_PTYPE_INNER_L4_TCP;
170 (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
171 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
172 RTE_PTYPE_INNER_L4_TCP;
173 (*p)[0x4d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
174 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
175 RTE_PTYPE_INNER_L4_TCP;
176 (*p)[0x4e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
177 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
178 RTE_PTYPE_INNER_L4_TCP;
179 (*p)[0x51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
180 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
181 RTE_PTYPE_INNER_L4_TCP;
182 (*p)[0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
183 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
184 RTE_PTYPE_INNER_L4_TCP;
185 (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
186 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
187 RTE_PTYPE_INNER_L4_TCP;
188 (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
189 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
190 RTE_PTYPE_INNER_L4_TCP;
191 (*p)[0xcd] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
192 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
193 RTE_PTYPE_INNER_L4_TCP;
194 (*p)[0xce] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
195 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
196 RTE_PTYPE_INNER_L4_TCP;
197 (*p)[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
198 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
199 RTE_PTYPE_INNER_L4_TCP;
200 (*p)[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
201 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
202 RTE_PTYPE_INNER_L4_TCP;
204 (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
205 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
206 RTE_PTYPE_INNER_L4_UDP;
207 (*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
208 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
209 RTE_PTYPE_INNER_L4_UDP;
210 (*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
211 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
212 RTE_PTYPE_INNER_L4_UDP;
213 (*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
214 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
215 RTE_PTYPE_INNER_L4_UDP;
219 * Build a table to translate packet to checksum type of Verbs.
222 mlx5_set_cksum_table(void)
228 * The index should have:
229 * bit[0] = PKT_TX_TCP_SEG
230 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
231 * bit[4] = PKT_TX_IP_CKSUM
232 * bit[8] = PKT_TX_OUTER_IP_CKSUM
235 for (i = 0; i < RTE_DIM(mlx5_cksum_table); ++i) {
238 /* Tunneled packet. */
239 if (i & (1 << 8)) /* Outer IP. */
240 v |= MLX5_ETH_WQE_L3_CSUM;
241 if (i & (1 << 4)) /* Inner IP. */
242 v |= MLX5_ETH_WQE_L3_INNER_CSUM;
243 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
244 v |= MLX5_ETH_WQE_L4_INNER_CSUM;
247 if (i & (1 << 4)) /* IP. */
248 v |= MLX5_ETH_WQE_L3_CSUM;
249 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
250 v |= MLX5_ETH_WQE_L4_CSUM;
252 mlx5_cksum_table[i] = v;
257 * Build a table to translate packet type of mbuf to SWP type of Verbs.
260 mlx5_set_swp_types_table(void)
266 * The index should have:
267 * bit[0:1] = PKT_TX_L4_MASK
268 * bit[4] = PKT_TX_IPV6
269 * bit[8] = PKT_TX_OUTER_IPV6
270 * bit[9] = PKT_TX_OUTER_UDP
272 for (i = 0; i < RTE_DIM(mlx5_swp_types_table); ++i) {
275 v |= MLX5_ETH_WQE_L3_OUTER_IPV6;
277 v |= MLX5_ETH_WQE_L4_OUTER_UDP;
279 v |= MLX5_ETH_WQE_L3_INNER_IPV6;
280 if ((i & 3) == (PKT_TX_UDP_CKSUM >> 52))
281 v |= MLX5_ETH_WQE_L4_INNER_UDP;
282 mlx5_swp_types_table[i] = v;
287 * Return the size of tailroom of WQ.
290 * Pointer to TX queue structure.
292 * Pointer to tail of WQ.
298 tx_mlx5_wq_tailroom(struct mlx5_txq_data *txq, void *addr)
301 tailroom = (uintptr_t)(txq->wqes) +
302 (1 << txq->wqe_n) * MLX5_WQE_SIZE -
308 * Copy data to tailroom of circular queue.
311 * Pointer to destination.
315 * Number of bytes to copy.
317 * Pointer to head of queue.
319 * Size of tailroom from dst.
322 * Pointer after copied data.
325 mlx5_copy_to_wq(void *dst, const void *src, size_t n,
326 void *base, size_t tailroom)
331 rte_memcpy(dst, src, tailroom);
332 rte_memcpy(base, (void *)((uintptr_t)src + tailroom),
334 ret = (uint8_t *)base + n - tailroom;
336 rte_memcpy(dst, src, n);
337 ret = (n == tailroom) ? base : (uint8_t *)dst + n;
343 * Inline TSO headers into WQE.
346 * 0 on success, negative errno value on failure.
349 inline_tso(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
352 uint16_t *pkt_inline_sz,
356 uint16_t *tso_header_sz)
358 uintptr_t end = (uintptr_t)(((uintptr_t)txq->wqes) +
359 (1 << txq->wqe_n) * MLX5_WQE_SIZE);
361 uint8_t vlan_sz = (buf->ol_flags & PKT_TX_VLAN_PKT) ? 4 : 0;
362 const uint8_t tunneled = txq->tunnel_en && (buf->ol_flags &
366 *tso_segsz = buf->tso_segsz;
367 *tso_header_sz = buf->l2_len + vlan_sz + buf->l3_len + buf->l4_len;
368 if (unlikely(*tso_segsz == 0 || *tso_header_sz == 0)) {
369 txq->stats.oerrors++;
373 *tso_header_sz += buf->outer_l2_len + buf->outer_l3_len;
374 /* First seg must contain all TSO headers. */
375 if (unlikely(*tso_header_sz > MLX5_MAX_TSO_HEADER) ||
376 *tso_header_sz > DATA_LEN(buf)) {
377 txq->stats.oerrors++;
380 copy_b = *tso_header_sz - *pkt_inline_sz;
381 if (!copy_b || ((end - (uintptr_t)*raw) < copy_b))
383 n_wqe = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4;
384 if (unlikely(*max_wqe < n_wqe))
387 rte_memcpy((void *)*raw, (void *)*addr, copy_b);
390 copy_b = MLX5_WQE_DS(copy_b) * MLX5_WQE_DWORD_SIZE;
391 *pkt_inline_sz += copy_b;
397 * DPDK callback to check the status of a tx descriptor.
402 * The index of the descriptor in the ring.
405 * The status of the tx descriptor.
408 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
410 struct mlx5_txq_data *txq = tx_queue;
413 mlx5_tx_complete(txq);
414 used = txq->elts_head - txq->elts_tail;
416 return RTE_ETH_TX_DESC_FULL;
417 return RTE_ETH_TX_DESC_DONE;
421 * Internal function to compute the number of used descriptors in an RX queue
427 * The number of used rx descriptor.
430 rx_queue_count(struct mlx5_rxq_data *rxq)
432 struct rxq_zip *zip = &rxq->zip;
433 volatile struct mlx5_cqe *cqe;
434 const unsigned int cqe_n = (1 << rxq->cqe_n);
435 const unsigned int cqe_cnt = cqe_n - 1;
439 /* if we are processing a compressed cqe */
441 used = zip->cqe_cnt - zip->ca;
447 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
448 while (check_cqe(cqe, cqe_n, cq_ci) != MLX5_CQE_STATUS_HW_OWN) {
452 op_own = cqe->op_own;
453 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED)
454 n = rte_be_to_cpu_32(cqe->byte_cnt);
459 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
461 used = RTE_MIN(used, (1U << rxq->elts_n) - 1);
466 * DPDK callback to check the status of a rx descriptor.
471 * The index of the descriptor in the ring.
474 * The status of the tx descriptor.
477 mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset)
479 struct mlx5_rxq_data *rxq = rx_queue;
480 struct mlx5_rxq_ctrl *rxq_ctrl =
481 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
482 struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv);
484 if (dev->rx_pkt_burst != mlx5_rx_burst) {
488 if (offset >= (1 << rxq->elts_n)) {
492 if (offset < rx_queue_count(rxq))
493 return RTE_ETH_RX_DESC_DONE;
494 return RTE_ETH_RX_DESC_AVAIL;
498 * DPDK callback to get the number of used descriptors in a RX queue
501 * Pointer to the device structure.
507 * The number of used rx descriptor.
508 * -EINVAL if the queue is invalid
511 mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
513 struct mlx5_priv *priv = dev->data->dev_private;
514 struct mlx5_rxq_data *rxq;
516 if (dev->rx_pkt_burst != mlx5_rx_burst) {
520 rxq = (*priv->rxqs)[rx_queue_id];
525 return rx_queue_count(rxq);
528 #define MLX5_SYSTEM_LOG_DIR "/var/log"
530 * Dump debug information to log file.
535 * If not NULL this string is printed as a header to the output
536 * and the output will be in hexadecimal view.
538 * This is the buffer address to print out.
540 * The number of bytes to dump out.
543 mlx5_dump_debug_information(const char *fname, const char *hex_title,
544 const void *buf, unsigned int hex_len)
548 MKSTR(path, "%s/%s", MLX5_SYSTEM_LOG_DIR, fname);
549 fd = fopen(path, "a+");
551 DRV_LOG(WARNING, "cannot open %s for debug dump\n",
553 MKSTR(path2, "./%s", fname);
554 fd = fopen(path2, "a+");
556 DRV_LOG(ERR, "cannot open %s for debug dump\n",
560 DRV_LOG(INFO, "New debug dump in file %s\n", path2);
562 DRV_LOG(INFO, "New debug dump in file %s\n", path);
565 rte_hexdump(fd, hex_title, buf, hex_len);
567 fprintf(fd, "%s", (const char *)buf);
568 fprintf(fd, "\n\n\n");
573 * Move QP from error state to running state.
576 * Pointer to TX queue structure.
578 * The qp pointer for recovery.
581 * 0 on success, else errno value.
584 tx_recover_qp(struct mlx5_txq_data *txq, struct ibv_qp *qp)
587 struct ibv_qp_attr mod = {
588 .qp_state = IBV_QPS_RESET,
591 ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
593 DRV_LOG(ERR, "Cannot change the Tx QP state to RESET %d\n",
597 mod.qp_state = IBV_QPS_INIT;
598 ret = mlx5_glue->modify_qp(qp, &mod,
599 (IBV_QP_STATE | IBV_QP_PORT));
601 DRV_LOG(ERR, "Cannot change Tx QP state to INIT %d\n", ret);
604 mod.qp_state = IBV_QPS_RTR;
605 ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
607 DRV_LOG(ERR, "Cannot change Tx QP state to RTR %d\n", ret);
610 mod.qp_state = IBV_QPS_RTS;
611 ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
613 DRV_LOG(ERR, "Cannot change Tx QP state to RTS %d\n", ret);
622 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */
624 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe)
626 static const uint8_t magic[] = "seen";
630 for (i = 0; i < sizeof(magic); ++i)
631 if (!ret || err_cqe->rsvd1[i] != magic[i]) {
633 err_cqe->rsvd1[i] = magic[i];
642 * Pointer to TX queue structure.
644 * Pointer to the error CQE.
647 * The last Tx buffer element to free.
650 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *txq,
651 volatile struct mlx5_err_cqe *err_cqe)
653 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {
654 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1);
655 struct mlx5_txq_ctrl *txq_ctrl =
656 container_of(txq, struct mlx5_txq_ctrl, txq);
657 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter);
658 int seen = check_err_cqe_seen(err_cqe);
660 if (!seen && txq_ctrl->dump_file_n <
661 txq_ctrl->priv->config.max_dump_files_num) {
662 MKSTR(err_str, "Unexpected CQE error syndrome "
663 "0x%02x CQN = %u SQN = %u wqe_counter = %u "
664 "wq_ci = %u cq_ci = %u", err_cqe->syndrome,
665 txq_ctrl->cqn, txq->qp_num_8s >> 8,
666 rte_be_to_cpu_16(err_cqe->wqe_counter),
667 txq->wqe_ci, txq->cq_ci);
668 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u",
669 PORT_ID(txq_ctrl->priv), txq->idx,
670 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc());
671 mlx5_dump_debug_information(name, NULL, err_str, 0);
672 mlx5_dump_debug_information(name, "MLX5 Error CQ:",
673 (const void *)((uintptr_t)
677 mlx5_dump_debug_information(name, "MLX5 Error SQ:",
678 (const void *)((uintptr_t)
679 tx_mlx5_wqe(txq, 0)),
682 txq_ctrl->dump_file_n++;
686 * Count errors in WQEs units.
687 * Later it can be improved to count error packets,
688 * for example, by SQ parsing to find how much packets
689 * should be counted for each WQE.
691 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) -
693 if ((rte_eal_process_type() == RTE_PROC_PRIMARY) &&
694 tx_recover_qp(txq, txq_ctrl->ibv->qp) == 0) {
696 /* Release all the remaining buffers. */
697 return txq->elts_head;
699 /* Recovering failed - try again later on the same WQE. */
703 /* Do not release buffers. */
704 return txq->elts_tail;
708 * DPDK callback for TX.
711 * Generic pointer to TX queue structure.
713 * Packets to transmit.
715 * Number of packets in array.
718 * Number of packets successfully transmitted (<= pkts_n).
721 mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
723 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
724 uint16_t elts_head = txq->elts_head;
725 const uint16_t elts_n = 1 << txq->elts_n;
726 const uint16_t elts_m = elts_n - 1;
733 volatile struct mlx5_wqe_ctrl *last_wqe = NULL;
734 unsigned int segs_n = 0;
735 const unsigned int max_inline = txq->max_inline;
738 if (unlikely(!pkts_n))
740 /* Prefetch first packet cacheline. */
741 rte_prefetch0(*pkts);
742 /* Start processing. */
743 mlx5_tx_complete(txq);
744 max_elts = (elts_n - (elts_head - txq->elts_tail));
745 max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
746 if (unlikely(!max_wqe))
749 struct rte_mbuf *buf = *pkts; /* First_seg. */
751 volatile struct mlx5_wqe_v *wqe = NULL;
752 volatile rte_v128u32_t *dseg = NULL;
755 unsigned int sg = 0; /* counter of additional segs attached. */
757 uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2;
758 uint16_t tso_header_sz = 0;
761 uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
762 uint32_t swp_offsets = 0;
763 uint8_t swp_types = 0;
765 uint16_t tso_segsz = 0;
766 #ifdef MLX5_PMD_SOFT_COUNTERS
767 uint32_t total_length = 0;
771 segs_n = buf->nb_segs;
773 * Make sure there is enough room to store this packet and
774 * that one ring entry remains unused.
777 if (max_elts < segs_n)
781 if (unlikely(--max_wqe == 0))
783 wqe = (volatile struct mlx5_wqe_v *)
784 tx_mlx5_wqe(txq, txq->wqe_ci);
785 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
787 rte_prefetch0(*(pkts + 1));
788 addr = rte_pktmbuf_mtod(buf, uintptr_t);
789 length = DATA_LEN(buf);
790 ehdr = (((uint8_t *)addr)[1] << 8) |
791 ((uint8_t *)addr)[0];
792 #ifdef MLX5_PMD_SOFT_COUNTERS
793 total_length = length;
795 if (length < (MLX5_WQE_DWORD_SIZE + 2)) {
796 txq->stats.oerrors++;
799 /* Update element. */
800 (*txq->elts)[elts_head & elts_m] = buf;
801 /* Prefetch next buffer data. */
804 rte_pktmbuf_mtod(*(pkts + 1), volatile void *));
805 cs_flags = txq_ol_cksum_to_cs(buf);
806 txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types);
807 raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
808 /* Copy metadata from mbuf if valid */
809 metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
811 /* Replace the Ethernet type by the VLAN if necessary. */
812 if (buf->ol_flags & PKT_TX_VLAN_PKT) {
813 uint32_t vlan = rte_cpu_to_be_32(0x81000000 |
815 unsigned int len = 2 * RTE_ETHER_ADDR_LEN - 2;
819 /* Copy Destination and source mac address. */
820 memcpy((uint8_t *)raw, ((uint8_t *)addr), len);
822 memcpy((uint8_t *)raw + len, &vlan, sizeof(vlan));
823 /* Copy missing two bytes to end the DSeg. */
824 memcpy((uint8_t *)raw + len + sizeof(vlan),
825 ((uint8_t *)addr) + len, 2);
829 memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2,
830 MLX5_WQE_DWORD_SIZE);
831 length -= pkt_inline_sz;
832 addr += pkt_inline_sz;
834 raw += MLX5_WQE_DWORD_SIZE;
836 ret = inline_tso(txq, buf, &length,
837 &addr, &pkt_inline_sz,
839 &tso_segsz, &tso_header_sz);
840 if (ret == -EINVAL) {
842 } else if (ret == -EAGAIN) {
844 wqe->ctrl = (rte_v128u32_t){
845 rte_cpu_to_be_32(txq->wqe_ci << 8),
846 rte_cpu_to_be_32(txq->qp_num_8s | 1),
848 (MLX5_COMP_ONLY_FIRST_ERR <<
849 MLX5_COMP_MODE_OFFSET),
853 #ifdef MLX5_PMD_SOFT_COUNTERS
860 /* Inline if enough room. */
861 if (max_inline || tso) {
863 uintptr_t end = (uintptr_t)
864 (((uintptr_t)txq->wqes) +
865 (1 << txq->wqe_n) * MLX5_WQE_SIZE);
866 unsigned int inline_room = max_inline *
867 RTE_CACHE_LINE_SIZE -
868 (pkt_inline_sz - 2) -
874 addr_end = RTE_ALIGN_FLOOR(addr + inline_room,
875 RTE_CACHE_LINE_SIZE);
876 copy_b = (addr_end > addr) ?
877 RTE_MIN((addr_end - addr), length) : 0;
878 if (copy_b && ((end - (uintptr_t)raw) >
879 (copy_b + sizeof(inl)))) {
881 * One Dseg remains in the current WQE. To
882 * keep the computation positive, it is
883 * removed after the bytes to Dseg conversion.
885 uint16_t n = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4;
887 if (unlikely(max_wqe < n))
892 inl = rte_cpu_to_be_32(copy_b |
894 rte_memcpy((void *)raw,
895 (void *)&inl, sizeof(inl));
897 pkt_inline_sz += sizeof(inl);
899 rte_memcpy((void *)raw, (void *)addr, copy_b);
902 pkt_inline_sz += copy_b;
905 * 2 DWORDs consumed by the WQE header + ETH segment +
906 * the size of the inline part of the packet.
908 ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2);
910 if (ds % (MLX5_WQE_SIZE /
911 MLX5_WQE_DWORD_SIZE) == 0) {
912 if (unlikely(--max_wqe == 0))
914 dseg = (volatile rte_v128u32_t *)
915 tx_mlx5_wqe(txq, txq->wqe_ci +
918 dseg = (volatile rte_v128u32_t *)
920 (ds * MLX5_WQE_DWORD_SIZE));
923 } else if (!segs_n) {
927 * Further inline the next segment only for
932 inline_room -= copy_b;
936 /* Move to the next segment. */
940 addr = rte_pktmbuf_mtod(buf, uintptr_t);
941 length = DATA_LEN(buf);
942 #ifdef MLX5_PMD_SOFT_COUNTERS
943 total_length += length;
945 (*txq->elts)[++elts_head & elts_m] = buf;
950 * No inline has been done in the packet, only the
951 * Ethernet Header as been stored.
953 dseg = (volatile rte_v128u32_t *)
954 ((uintptr_t)wqe + (3 * MLX5_WQE_DWORD_SIZE));
957 /* Add the remaining packet as a simple ds. */
958 addr_64 = rte_cpu_to_be_64(addr);
959 *dseg = (rte_v128u32_t){
960 rte_cpu_to_be_32(length),
961 mlx5_tx_mb2mr(txq, buf),
974 * Spill on next WQE when the current one does not have
975 * enough room left. Size of WQE must a be a multiple
976 * of data segment size.
978 assert(!(MLX5_WQE_SIZE % MLX5_WQE_DWORD_SIZE));
979 if (!(ds % (MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE))) {
980 if (unlikely(--max_wqe == 0))
982 dseg = (volatile rte_v128u32_t *)
983 tx_mlx5_wqe(txq, txq->wqe_ci + ds / 4);
984 rte_prefetch0(tx_mlx5_wqe(txq,
985 txq->wqe_ci + ds / 4 + 1));
992 length = DATA_LEN(buf);
993 #ifdef MLX5_PMD_SOFT_COUNTERS
994 total_length += length;
996 /* Store segment information. */
997 addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t));
998 *dseg = (rte_v128u32_t){
999 rte_cpu_to_be_32(length),
1000 mlx5_tx_mb2mr(txq, buf),
1004 (*txq->elts)[++elts_head & elts_m] = buf;
1008 if (ds > MLX5_DSEG_MAX) {
1009 txq->stats.oerrors++;
1016 /* Initialize known and common part of the WQE structure. */
1018 wqe->ctrl = (rte_v128u32_t){
1019 rte_cpu_to_be_32((txq->wqe_ci << 8) |
1021 rte_cpu_to_be_32(txq->qp_num_8s | ds),
1022 rte_cpu_to_be_32(MLX5_COMP_ONLY_FIRST_ERR <<
1023 MLX5_COMP_MODE_OFFSET),
1026 wqe->eseg = (rte_v128u32_t){
1028 cs_flags | (swp_types << 8) |
1029 (rte_cpu_to_be_16(tso_segsz) << 16),
1031 (ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
1034 wqe->ctrl = (rte_v128u32_t){
1035 rte_cpu_to_be_32((txq->wqe_ci << 8) |
1037 rte_cpu_to_be_32(txq->qp_num_8s | ds),
1038 rte_cpu_to_be_32(MLX5_COMP_ONLY_FIRST_ERR <<
1039 MLX5_COMP_MODE_OFFSET),
1042 wqe->eseg = (rte_v128u32_t){
1044 cs_flags | (swp_types << 8),
1046 (ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
1050 txq->wqe_ci += (ds + 3) / 4;
1051 /* Save the last successful WQE for completion request */
1052 last_wqe = (volatile struct mlx5_wqe_ctrl *)wqe;
1053 #ifdef MLX5_PMD_SOFT_COUNTERS
1054 /* Increment sent bytes counter. */
1055 txq->stats.obytes += total_length;
1057 } while (i < pkts_n);
1058 /* Take a shortcut if nothing must be sent. */
1059 if (unlikely((i + k) == 0))
1061 txq->elts_head += (i + j);
1062 /* Check whether completion threshold has been reached. */
1063 comp = txq->elts_comp + i + j + k;
1064 if (comp >= MLX5_TX_COMP_THRESH) {
1065 /* A CQE slot must always be available. */
1066 assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
1067 /* Request completion on last WQE. */
1068 last_wqe->ctrl2 = rte_cpu_to_be_32(MLX5_COMP_ALWAYS <<
1069 MLX5_COMP_MODE_OFFSET);
1070 /* Save elts_head in unused "immediate" field of WQE. */
1071 last_wqe->ctrl3 = txq->elts_head;
1074 txq->elts_comp = comp;
1076 #ifdef MLX5_PMD_SOFT_COUNTERS
1077 /* Increment sent packets counter. */
1078 txq->stats.opackets += i;
1080 /* Ring QP doorbell. */
1081 mlx5_tx_dbrec(txq, (volatile struct mlx5_wqe *)last_wqe);
1086 * Open a MPW session.
1089 * Pointer to TX queue structure.
1091 * Pointer to MPW session structure.
1096 mlx5_mpw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, uint32_t length)
1098 uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
1099 volatile struct mlx5_wqe_data_seg (*dseg)[MLX5_MPW_DSEG_MAX] =
1100 (volatile struct mlx5_wqe_data_seg (*)[])
1101 tx_mlx5_wqe(txq, idx + 1);
1103 mpw->state = MLX5_MPW_STATE_OPENED;
1107 mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx);
1108 mpw->wqe->eseg.mss = rte_cpu_to_be_16(length);
1109 mpw->wqe->eseg.inline_hdr_sz = 0;
1110 mpw->wqe->eseg.rsvd0 = 0;
1111 mpw->wqe->eseg.rsvd1 = 0;
1112 mpw->wqe->eseg.flow_table_metadata = 0;
1113 mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
1114 (txq->wqe_ci << 8) |
1116 mpw->wqe->ctrl[2] = rte_cpu_to_be_32(MLX5_COMP_ONLY_FIRST_ERR <<
1117 MLX5_COMP_MODE_OFFSET);
1118 mpw->wqe->ctrl[3] = 0;
1119 mpw->data.dseg[0] = (volatile struct mlx5_wqe_data_seg *)
1120 (((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE));
1121 mpw->data.dseg[1] = (volatile struct mlx5_wqe_data_seg *)
1122 (((uintptr_t)mpw->wqe) + (3 * MLX5_WQE_DWORD_SIZE));
1123 mpw->data.dseg[2] = &(*dseg)[0];
1124 mpw->data.dseg[3] = &(*dseg)[1];
1125 mpw->data.dseg[4] = &(*dseg)[2];
1129 * Close a MPW session.
1132 * Pointer to TX queue structure.
1134 * Pointer to MPW session structure.
1137 mlx5_mpw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
1139 unsigned int num = mpw->pkts_n;
1142 * Store size in multiple of 16 bytes. Control and Ethernet segments
1145 mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s | (2 + num));
1146 mpw->state = MLX5_MPW_STATE_CLOSED;
1151 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
1152 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
1156 * DPDK callback for TX with MPW support.
1159 * Generic pointer to TX queue structure.
1161 * Packets to transmit.
1163 * Number of packets in array.
1166 * Number of packets successfully transmitted (<= pkts_n).
1169 mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
1171 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
1172 uint16_t elts_head = txq->elts_head;
1173 const uint16_t elts_n = 1 << txq->elts_n;
1174 const uint16_t elts_m = elts_n - 1;
1180 struct mlx5_mpw mpw = {
1181 .state = MLX5_MPW_STATE_CLOSED,
1184 if (unlikely(!pkts_n))
1186 /* Prefetch first packet cacheline. */
1187 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
1188 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
1189 /* Start processing. */
1190 mlx5_tx_complete(txq);
1191 max_elts = (elts_n - (elts_head - txq->elts_tail));
1192 max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
1193 if (unlikely(!max_wqe))
1196 struct rte_mbuf *buf = *(pkts++);
1198 unsigned int segs_n = buf->nb_segs;
1200 rte_be32_t metadata;
1203 * Make sure there is enough room to store this packet and
1204 * that one ring entry remains unused.
1207 if (max_elts < segs_n)
1209 /* Do not bother with large packets MPW cannot handle. */
1210 if (segs_n > MLX5_MPW_DSEG_MAX) {
1211 txq->stats.oerrors++;
1216 cs_flags = txq_ol_cksum_to_cs(buf);
1217 /* Copy metadata from mbuf if valid */
1218 metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
1220 /* Retrieve packet information. */
1221 length = PKT_LEN(buf);
1223 /* Start new session if packet differs. */
1224 if ((mpw.state == MLX5_MPW_STATE_OPENED) &&
1225 ((mpw.len != length) ||
1227 (mpw.wqe->eseg.flow_table_metadata != metadata) ||
1228 (mpw.wqe->eseg.cs_flags != cs_flags)))
1229 mlx5_mpw_close(txq, &mpw);
1230 if (mpw.state == MLX5_MPW_STATE_CLOSED) {
1232 * Multi-Packet WQE consumes at most two WQE.
1233 * mlx5_mpw_new() expects to be able to use such
1236 if (unlikely(max_wqe < 2))
1239 mlx5_mpw_new(txq, &mpw, length);
1240 mpw.wqe->eseg.cs_flags = cs_flags;
1241 mpw.wqe->eseg.flow_table_metadata = metadata;
1243 /* Multi-segment packets must be alone in their MPW. */
1244 assert((segs_n == 1) || (mpw.pkts_n == 0));
1245 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
1249 volatile struct mlx5_wqe_data_seg *dseg;
1253 (*txq->elts)[elts_head++ & elts_m] = buf;
1254 dseg = mpw.data.dseg[mpw.pkts_n];
1255 addr = rte_pktmbuf_mtod(buf, uintptr_t);
1256 *dseg = (struct mlx5_wqe_data_seg){
1257 .byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
1258 .lkey = mlx5_tx_mb2mr(txq, buf),
1259 .addr = rte_cpu_to_be_64(addr),
1261 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
1262 length += DATA_LEN(buf);
1268 assert(length == mpw.len);
1269 if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
1270 mlx5_mpw_close(txq, &mpw);
1271 #ifdef MLX5_PMD_SOFT_COUNTERS
1272 /* Increment sent bytes counter. */
1273 txq->stats.obytes += length;
1277 /* Take a shortcut if nothing must be sent. */
1278 if (unlikely(i == 0))
1280 /* Check whether completion threshold has been reached. */
1281 /* "j" includes both packets and segments. */
1282 comp = txq->elts_comp + j;
1283 if (comp >= MLX5_TX_COMP_THRESH) {
1284 volatile struct mlx5_wqe *wqe = mpw.wqe;
1286 /* A CQE slot must always be available. */
1287 assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
1288 /* Request completion on last WQE. */
1289 wqe->ctrl[2] = rte_cpu_to_be_32(MLX5_COMP_ALWAYS <<
1290 MLX5_COMP_MODE_OFFSET);
1291 /* Save elts_head in unused "immediate" field of WQE. */
1292 wqe->ctrl[3] = elts_head;
1295 txq->elts_comp = comp;
1297 #ifdef MLX5_PMD_SOFT_COUNTERS
1298 /* Increment sent packets counter. */
1299 txq->stats.opackets += i;
1301 /* Ring QP doorbell. */
1302 if (mpw.state == MLX5_MPW_STATE_OPENED)
1303 mlx5_mpw_close(txq, &mpw);
1304 mlx5_tx_dbrec(txq, mpw.wqe);
1305 txq->elts_head = elts_head;
1310 * Open a MPW inline session.
1313 * Pointer to TX queue structure.
1315 * Pointer to MPW session structure.
1320 mlx5_mpw_inline_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw,
1323 uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
1324 struct mlx5_wqe_inl_small *inl;
1326 mpw->state = MLX5_MPW_INL_STATE_OPENED;
1330 mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx);
1331 mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
1332 (txq->wqe_ci << 8) |
1334 mpw->wqe->ctrl[2] = rte_cpu_to_be_32(MLX5_COMP_ONLY_FIRST_ERR <<
1335 MLX5_COMP_MODE_OFFSET);
1336 mpw->wqe->ctrl[3] = 0;
1337 mpw->wqe->eseg.mss = rte_cpu_to_be_16(length);
1338 mpw->wqe->eseg.inline_hdr_sz = 0;
1339 mpw->wqe->eseg.cs_flags = 0;
1340 mpw->wqe->eseg.rsvd0 = 0;
1341 mpw->wqe->eseg.rsvd1 = 0;
1342 mpw->wqe->eseg.flow_table_metadata = 0;
1343 inl = (struct mlx5_wqe_inl_small *)
1344 (((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE);
1345 mpw->data.raw = (uint8_t *)&inl->raw;
1349 * Close a MPW inline session.
1352 * Pointer to TX queue structure.
1354 * Pointer to MPW session structure.
1357 mlx5_mpw_inline_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
1360 struct mlx5_wqe_inl_small *inl = (struct mlx5_wqe_inl_small *)
1361 (((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE));
1363 size = MLX5_WQE_SIZE - MLX5_MWQE64_INL_DATA + mpw->total_len;
1365 * Store size in multiple of 16 bytes. Control and Ethernet segments
1368 mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s |
1370 mpw->state = MLX5_MPW_STATE_CLOSED;
1371 inl->byte_cnt = rte_cpu_to_be_32(mpw->total_len | MLX5_INLINE_SEG);
1372 txq->wqe_ci += (size + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE;
1376 * DPDK callback for TX with MPW inline support.
1379 * Generic pointer to TX queue structure.
1381 * Packets to transmit.
1383 * Number of packets in array.
1386 * Number of packets successfully transmitted (<= pkts_n).
1389 mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
1392 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
1393 uint16_t elts_head = txq->elts_head;
1394 const uint16_t elts_n = 1 << txq->elts_n;
1395 const uint16_t elts_m = elts_n - 1;
1401 unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
1402 struct mlx5_mpw mpw = {
1403 .state = MLX5_MPW_STATE_CLOSED,
1406 * Compute the maximum number of WQE which can be consumed by inline
1409 * - 1 control segment,
1410 * - 1 Ethernet segment,
1411 * - N Dseg from the inline request.
1413 const unsigned int wqe_inl_n =
1414 ((2 * MLX5_WQE_DWORD_SIZE +
1415 txq->max_inline * RTE_CACHE_LINE_SIZE) +
1416 RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE;
1418 if (unlikely(!pkts_n))
1420 /* Prefetch first packet cacheline. */
1421 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
1422 rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
1423 /* Start processing. */
1424 mlx5_tx_complete(txq);
1425 max_elts = (elts_n - (elts_head - txq->elts_tail));
1427 struct rte_mbuf *buf = *(pkts++);
1430 unsigned int segs_n = buf->nb_segs;
1432 rte_be32_t metadata;
1435 * Make sure there is enough room to store this packet and
1436 * that one ring entry remains unused.
1439 if (max_elts < segs_n)
1441 /* Do not bother with large packets MPW cannot handle. */
1442 if (segs_n > MLX5_MPW_DSEG_MAX) {
1443 txq->stats.oerrors++;
1449 * Compute max_wqe in case less WQE were consumed in previous
1452 max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
1453 cs_flags = txq_ol_cksum_to_cs(buf);
1454 /* Copy metadata from mbuf if valid */
1455 metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
1457 /* Retrieve packet information. */
1458 length = PKT_LEN(buf);
1459 /* Start new session if packet differs. */
1460 if (mpw.state == MLX5_MPW_STATE_OPENED) {
1461 if ((mpw.len != length) ||
1463 (mpw.wqe->eseg.flow_table_metadata != metadata) ||
1464 (mpw.wqe->eseg.cs_flags != cs_flags))
1465 mlx5_mpw_close(txq, &mpw);
1466 } else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) {
1467 if ((mpw.len != length) ||
1469 (length > inline_room) ||
1470 (mpw.wqe->eseg.flow_table_metadata != metadata) ||
1471 (mpw.wqe->eseg.cs_flags != cs_flags)) {
1472 mlx5_mpw_inline_close(txq, &mpw);
1474 txq->max_inline * RTE_CACHE_LINE_SIZE;
1477 if (mpw.state == MLX5_MPW_STATE_CLOSED) {
1478 if ((segs_n != 1) ||
1479 (length > inline_room)) {
1481 * Multi-Packet WQE consumes at most two WQE.
1482 * mlx5_mpw_new() expects to be able to use
1485 if (unlikely(max_wqe < 2))
1488 mlx5_mpw_new(txq, &mpw, length);
1489 mpw.wqe->eseg.cs_flags = cs_flags;
1490 mpw.wqe->eseg.flow_table_metadata = metadata;
1492 if (unlikely(max_wqe < wqe_inl_n))
1494 max_wqe -= wqe_inl_n;
1495 mlx5_mpw_inline_new(txq, &mpw, length);
1496 mpw.wqe->eseg.cs_flags = cs_flags;
1497 mpw.wqe->eseg.flow_table_metadata = metadata;
1500 /* Multi-segment packets must be alone in their MPW. */
1501 assert((segs_n == 1) || (mpw.pkts_n == 0));
1502 if (mpw.state == MLX5_MPW_STATE_OPENED) {
1503 assert(inline_room ==
1504 txq->max_inline * RTE_CACHE_LINE_SIZE);
1505 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
1509 volatile struct mlx5_wqe_data_seg *dseg;
1512 (*txq->elts)[elts_head++ & elts_m] = buf;
1513 dseg = mpw.data.dseg[mpw.pkts_n];
1514 addr = rte_pktmbuf_mtod(buf, uintptr_t);
1515 *dseg = (struct mlx5_wqe_data_seg){
1517 rte_cpu_to_be_32(DATA_LEN(buf)),
1518 .lkey = mlx5_tx_mb2mr(txq, buf),
1519 .addr = rte_cpu_to_be_64(addr),
1521 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
1522 length += DATA_LEN(buf);
1528 assert(length == mpw.len);
1529 if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
1530 mlx5_mpw_close(txq, &mpw);
1534 assert(mpw.state == MLX5_MPW_INL_STATE_OPENED);
1535 assert(length <= inline_room);
1536 assert(length == DATA_LEN(buf));
1537 addr = rte_pktmbuf_mtod(buf, uintptr_t);
1538 (*txq->elts)[elts_head++ & elts_m] = buf;
1539 /* Maximum number of bytes before wrapping. */
1540 max = ((((uintptr_t)(txq->wqes)) +
1543 (uintptr_t)mpw.data.raw);
1545 rte_memcpy((void *)(uintptr_t)mpw.data.raw,
1548 mpw.data.raw = (volatile void *)txq->wqes;
1549 rte_memcpy((void *)(uintptr_t)mpw.data.raw,
1550 (void *)(addr + max),
1552 mpw.data.raw += length - max;
1554 rte_memcpy((void *)(uintptr_t)mpw.data.raw,
1560 (volatile void *)txq->wqes;
1562 mpw.data.raw += length;
1565 mpw.total_len += length;
1567 if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) {
1568 mlx5_mpw_inline_close(txq, &mpw);
1570 txq->max_inline * RTE_CACHE_LINE_SIZE;
1572 inline_room -= length;
1575 #ifdef MLX5_PMD_SOFT_COUNTERS
1576 /* Increment sent bytes counter. */
1577 txq->stats.obytes += length;
1581 /* Take a shortcut if nothing must be sent. */
1582 if (unlikely(i == 0))
1584 /* Check whether completion threshold has been reached. */
1585 /* "j" includes both packets and segments. */
1586 comp = txq->elts_comp + j;
1587 if (comp >= MLX5_TX_COMP_THRESH) {
1588 volatile struct mlx5_wqe *wqe = mpw.wqe;
1590 /* A CQE slot must always be available. */
1591 assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
1592 /* Request completion on last WQE. */
1593 wqe->ctrl[2] = rte_cpu_to_be_32(MLX5_COMP_ALWAYS <<
1594 MLX5_COMP_MODE_OFFSET);
1595 /* Save elts_head in unused "immediate" field of WQE. */
1596 wqe->ctrl[3] = elts_head;
1599 txq->elts_comp = comp;
1601 #ifdef MLX5_PMD_SOFT_COUNTERS
1602 /* Increment sent packets counter. */
1603 txq->stats.opackets += i;
1605 /* Ring QP doorbell. */
1606 if (mpw.state == MLX5_MPW_INL_STATE_OPENED)
1607 mlx5_mpw_inline_close(txq, &mpw);
1608 else if (mpw.state == MLX5_MPW_STATE_OPENED)
1609 mlx5_mpw_close(txq, &mpw);
1610 mlx5_tx_dbrec(txq, mpw.wqe);
1611 txq->elts_head = elts_head;
1616 * Open an Enhanced MPW session.
1619 * Pointer to TX queue structure.
1621 * Pointer to MPW session structure.
1626 mlx5_empw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, int padding)
1628 uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
1630 mpw->state = MLX5_MPW_ENHANCED_STATE_OPENED;
1632 mpw->total_len = sizeof(struct mlx5_wqe);
1633 mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx);
1635 rte_cpu_to_be_32((MLX5_OPC_MOD_ENHANCED_MPSW << 24) |
1636 (txq->wqe_ci << 8) |
1637 MLX5_OPCODE_ENHANCED_MPSW);
1638 mpw->wqe->ctrl[2] = rte_cpu_to_be_32(MLX5_COMP_ONLY_FIRST_ERR <<
1639 MLX5_COMP_MODE_OFFSET);
1640 mpw->wqe->ctrl[3] = 0;
1641 memset((void *)(uintptr_t)&mpw->wqe->eseg, 0, MLX5_WQE_DWORD_SIZE);
1642 if (unlikely(padding)) {
1643 uintptr_t addr = (uintptr_t)(mpw->wqe + 1);
1645 /* Pad the first 2 DWORDs with zero-length inline header. */
1646 *(volatile uint32_t *)addr = rte_cpu_to_be_32(MLX5_INLINE_SEG);
1647 *(volatile uint32_t *)(addr + MLX5_WQE_DWORD_SIZE) =
1648 rte_cpu_to_be_32(MLX5_INLINE_SEG);
1649 mpw->total_len += 2 * MLX5_WQE_DWORD_SIZE;
1650 /* Start from the next WQEBB. */
1651 mpw->data.raw = (volatile void *)(tx_mlx5_wqe(txq, idx + 1));
1653 mpw->data.raw = (volatile void *)(mpw->wqe + 1);
1658 * Close an Enhanced MPW session.
1661 * Pointer to TX queue structure.
1663 * Pointer to MPW session structure.
1666 * Number of consumed WQEs.
1668 static inline uint16_t
1669 mlx5_empw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
1673 /* Store size in multiple of 16 bytes. Control and Ethernet segments
1676 mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s |
1677 MLX5_WQE_DS(mpw->total_len));
1678 mpw->state = MLX5_MPW_STATE_CLOSED;
1679 ret = (mpw->total_len + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE;
1685 * TX with Enhanced MPW support.
1688 * Pointer to TX queue structure.
1690 * Packets to transmit.
1692 * Number of packets in array.
1695 * Number of packets successfully transmitted (<= pkts_n).
1697 static inline uint16_t
1698 txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
1701 uint16_t elts_head = txq->elts_head;
1702 const uint16_t elts_n = 1 << txq->elts_n;
1703 const uint16_t elts_m = elts_n - 1;
1708 unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE;
1709 unsigned int mpw_room = 0;
1710 unsigned int inl_pad = 0;
1713 struct mlx5_mpw mpw = {
1714 .state = MLX5_MPW_STATE_CLOSED,
1717 if (unlikely(!pkts_n))
1719 /* Start processing. */
1720 mlx5_tx_complete(txq);
1721 max_elts = (elts_n - (elts_head - txq->elts_tail));
1722 max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
1723 if (unlikely(!max_wqe))
1726 struct rte_mbuf *buf = *(pkts++);
1728 unsigned int do_inline = 0; /* Whether inline is possible. */
1731 rte_be32_t metadata;
1733 /* Multi-segmented packet is handled in slow-path outside. */
1734 assert(NB_SEGS(buf) == 1);
1735 /* Make sure there is enough room to store this packet. */
1736 if (max_elts - j == 0)
1738 cs_flags = txq_ol_cksum_to_cs(buf);
1739 /* Copy metadata from mbuf if valid */
1740 metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
1742 /* Retrieve packet information. */
1743 length = PKT_LEN(buf);
1744 /* Start new session if:
1745 * - multi-segment packet
1746 * - no space left even for a dseg
1747 * - next packet can be inlined with a new WQE
1750 if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED) {
1751 if ((inl_pad + sizeof(struct mlx5_wqe_data_seg) >
1753 (length <= txq->inline_max_packet_sz &&
1754 inl_pad + sizeof(inl_hdr) + length >
1756 (mpw.wqe->eseg.flow_table_metadata != metadata) ||
1757 (mpw.wqe->eseg.cs_flags != cs_flags))
1758 max_wqe -= mlx5_empw_close(txq, &mpw);
1760 if (unlikely(mpw.state == MLX5_MPW_STATE_CLOSED)) {
1761 /* In Enhanced MPW, inline as much as the budget is
1762 * allowed. The remaining space is to be filled with
1763 * dsegs. If the title WQEBB isn't padded, it will have
1766 mpw_room = RTE_MIN(MLX5_WQE_SIZE_MAX,
1767 (max_inline ? max_inline :
1768 pkts_n * MLX5_WQE_DWORD_SIZE) +
1770 if (unlikely(max_wqe * MLX5_WQE_SIZE < mpw_room))
1772 /* Don't pad the title WQEBB to not waste WQ. */
1773 mlx5_empw_new(txq, &mpw, 0);
1774 mpw_room -= mpw.total_len;
1776 do_inline = length <= txq->inline_max_packet_sz &&
1777 sizeof(inl_hdr) + length <= mpw_room &&
1779 mpw.wqe->eseg.cs_flags = cs_flags;
1780 mpw.wqe->eseg.flow_table_metadata = metadata;
1782 /* Evaluate whether the next packet can be inlined.
1783 * Inlininig is possible when:
1784 * - length is less than configured value
1785 * - length fits for remaining space
1786 * - not required to fill the title WQEBB with dsegs
1789 length <= txq->inline_max_packet_sz &&
1790 inl_pad + sizeof(inl_hdr) + length <=
1792 (!txq->mpw_hdr_dseg ||
1793 mpw.total_len >= MLX5_WQE_SIZE);
1795 if (max_inline && do_inline) {
1796 /* Inline packet into WQE. */
1799 assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED);
1800 assert(length == DATA_LEN(buf));
1801 inl_hdr = rte_cpu_to_be_32(length | MLX5_INLINE_SEG);
1802 addr = rte_pktmbuf_mtod(buf, uintptr_t);
1803 mpw.data.raw = (volatile void *)
1804 ((uintptr_t)mpw.data.raw + inl_pad);
1805 max = tx_mlx5_wq_tailroom(txq,
1806 (void *)(uintptr_t)mpw.data.raw);
1807 /* Copy inline header. */
1808 mpw.data.raw = (volatile void *)
1810 (void *)(uintptr_t)mpw.data.raw,
1813 (void *)(uintptr_t)txq->wqes,
1815 max = tx_mlx5_wq_tailroom(txq,
1816 (void *)(uintptr_t)mpw.data.raw);
1817 /* Copy packet data. */
1818 mpw.data.raw = (volatile void *)
1820 (void *)(uintptr_t)mpw.data.raw,
1823 (void *)(uintptr_t)txq->wqes,
1826 mpw.total_len += (inl_pad + sizeof(inl_hdr) + length);
1827 /* No need to get completion as the entire packet is
1828 * copied to WQ. Free the buf right away.
1830 rte_pktmbuf_free_seg(buf);
1831 mpw_room -= (inl_pad + sizeof(inl_hdr) + length);
1832 /* Add pad in the next packet if any. */
1833 inl_pad = (((uintptr_t)mpw.data.raw +
1834 (MLX5_WQE_DWORD_SIZE - 1)) &
1835 ~(MLX5_WQE_DWORD_SIZE - 1)) -
1836 (uintptr_t)mpw.data.raw;
1838 /* No inline. Load a dseg of packet pointer. */
1839 volatile rte_v128u32_t *dseg;
1841 assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED);
1842 assert((inl_pad + sizeof(*dseg)) <= mpw_room);
1843 assert(length == DATA_LEN(buf));
1844 if (!tx_mlx5_wq_tailroom(txq,
1845 (void *)((uintptr_t)mpw.data.raw
1847 dseg = (volatile void *)txq->wqes;
1849 dseg = (volatile void *)
1850 ((uintptr_t)mpw.data.raw +
1852 (*txq->elts)[elts_head++ & elts_m] = buf;
1853 addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
1855 *dseg = (rte_v128u32_t) {
1856 rte_cpu_to_be_32(length),
1857 mlx5_tx_mb2mr(txq, buf),
1861 mpw.data.raw = (volatile void *)(dseg + 1);
1862 mpw.total_len += (inl_pad + sizeof(*dseg));
1865 mpw_room -= (inl_pad + sizeof(*dseg));
1868 #ifdef MLX5_PMD_SOFT_COUNTERS
1869 /* Increment sent bytes counter. */
1870 txq->stats.obytes += length;
1873 } while (i < pkts_n);
1874 /* Take a shortcut if nothing must be sent. */
1875 if (unlikely(i == 0))
1877 /* Check whether completion threshold has been reached. */
1878 if (txq->elts_comp + j >= MLX5_TX_COMP_THRESH ||
1879 (uint16_t)(txq->wqe_ci - txq->mpw_comp) >=
1880 (1 << txq->wqe_n) / MLX5_TX_COMP_THRESH_INLINE_DIV) {
1881 volatile struct mlx5_wqe *wqe = mpw.wqe;
1883 /* A CQE slot must always be available. */
1884 assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
1885 /* Request completion on last WQE. */
1886 wqe->ctrl[2] = rte_cpu_to_be_32(MLX5_COMP_ALWAYS <<
1887 MLX5_COMP_MODE_OFFSET);
1888 /* Save elts_head in unused "immediate" field of WQE. */
1889 wqe->ctrl[3] = elts_head;
1891 txq->mpw_comp = txq->wqe_ci;
1893 txq->elts_comp += j;
1895 #ifdef MLX5_PMD_SOFT_COUNTERS
1896 /* Increment sent packets counter. */
1897 txq->stats.opackets += i;
1899 if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED)
1900 mlx5_empw_close(txq, &mpw);
1901 /* Ring QP doorbell. */
1902 mlx5_tx_dbrec(txq, mpw.wqe);
1903 txq->elts_head = elts_head;
1908 * DPDK callback for TX with Enhanced MPW support.
1911 * Generic pointer to TX queue structure.
1913 * Packets to transmit.
1915 * Number of packets in array.
1918 * Number of packets successfully transmitted (<= pkts_n).
1921 mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
1923 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
1926 while (pkts_n > nb_tx) {
1930 n = txq_count_contig_multi_seg(&pkts[nb_tx], pkts_n - nb_tx);
1932 ret = mlx5_tx_burst(dpdk_txq, &pkts[nb_tx], n);
1937 n = txq_count_contig_single_seg(&pkts[nb_tx], pkts_n - nb_tx);
1939 ret = txq_burst_empw(txq, &pkts[nb_tx], n);
1949 * Translate RX completion flags to packet type.
1952 * Pointer to RX queue structure.
1956 * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
1959 * Packet type for struct rte_mbuf.
1961 static inline uint32_t
1962 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe)
1965 uint8_t pinfo = cqe->pkt_info;
1966 uint16_t ptype = cqe->hdr_type_etc;
1969 * The index to the array should have:
1970 * bit[1:0] = l3_hdr_type
1971 * bit[4:2] = l4_hdr_type
1974 * bit[7] = outer_l3_type
1976 idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10);
1977 return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
1981 * Initialize Rx WQ and indexes.
1984 * Pointer to RX queue structure.
1987 mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
1989 const unsigned int wqe_n = 1 << rxq->elts_n;
1992 for (i = 0; (i != wqe_n); ++i) {
1993 volatile struct mlx5_wqe_data_seg *scat;
1995 uint32_t byte_count;
1997 if (mlx5_rxq_mprq_enabled(rxq)) {
1998 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
2000 scat = &((volatile struct mlx5_wqe_mprq *)
2002 addr = (uintptr_t)mlx5_mprq_buf_addr(buf);
2003 byte_count = (1 << rxq->strd_sz_n) *
2004 (1 << rxq->strd_num_n);
2006 struct rte_mbuf *buf = (*rxq->elts)[i];
2008 scat = &((volatile struct mlx5_wqe_data_seg *)
2010 addr = rte_pktmbuf_mtod(buf, uintptr_t);
2011 byte_count = DATA_LEN(buf);
2013 /* scat->addr must be able to store a pointer. */
2014 assert(sizeof(scat->addr) >= sizeof(uintptr_t));
2015 *scat = (struct mlx5_wqe_data_seg){
2016 .addr = rte_cpu_to_be_64(addr),
2017 .byte_count = rte_cpu_to_be_32(byte_count),
2018 .lkey = mlx5_rx_addr2mr(rxq, addr),
2021 rxq->consumed_strd = 0;
2022 rxq->decompressed = 0;
2024 rxq->zip = (struct rxq_zip){
2027 /* Update doorbell counter. */
2028 rxq->rq_ci = wqe_n >> rxq->sges_n;
2030 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
2034 * Modify a Verbs queue state.
2035 * This must be called from the primary process.
2038 * Pointer to Ethernet device.
2040 * State modify request parameters.
2043 * 0 in case of success else non-zero value and rte_errno is set.
2046 mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
2047 const struct mlx5_mp_arg_queue_state_modify *sm)
2050 struct mlx5_priv *priv = dev->data->dev_private;
2053 struct ibv_wq_attr mod = {
2054 .attr_mask = IBV_WQ_ATTR_STATE,
2055 .wq_state = sm->state,
2057 struct mlx5_rxq_data *rxq = (*priv->rxqs)[sm->queue_id];
2058 struct mlx5_rxq_ctrl *rxq_ctrl =
2059 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
2061 ret = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod);
2063 DRV_LOG(ERR, "Cannot change Rx WQ state to %u - %s\n",
2064 sm->state, strerror(errno));
2073 * Modify a Verbs queue state.
2076 * Pointer to Ethernet device.
2078 * State modify request parameters.
2081 * 0 in case of success else non-zero value.
2084 mlx5_queue_state_modify(struct rte_eth_dev *dev,
2085 struct mlx5_mp_arg_queue_state_modify *sm)
2089 switch (rte_eal_process_type()) {
2090 case RTE_PROC_PRIMARY:
2091 ret = mlx5_queue_state_modify_primary(dev, sm);
2093 case RTE_PROC_SECONDARY:
2094 ret = mlx5_mp_req_queue_state_modify(dev, sm);
2103 * Handle a Rx error.
2104 * The function inserts the RQ state to reset when the first error CQE is
2105 * shown, then drains the CQ by the caller function loop. When the CQ is empty,
2106 * it moves the RQ state to ready and initializes the RQ.
2107 * Next CQE identification and error counting are in the caller responsibility.
2110 * Pointer to RX queue structure.
2111 * @param[in] mbuf_prepare
2112 * Whether to prepare mbufs for the RQ.
2115 * -1 in case of recovery error, otherwise the CQE status.
2118 mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t mbuf_prepare)
2120 const uint16_t cqe_n = 1 << rxq->cqe_n;
2121 const uint16_t cqe_mask = cqe_n - 1;
2122 const unsigned int wqe_n = 1 << rxq->elts_n;
2123 struct mlx5_rxq_ctrl *rxq_ctrl =
2124 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
2126 volatile struct mlx5_cqe *cqe;
2127 volatile struct mlx5_err_cqe *err_cqe;
2129 .cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask],
2131 struct mlx5_mp_arg_queue_state_modify sm;
2134 switch (rxq->err_state) {
2135 case MLX5_RXQ_ERR_STATE_NO_ERROR:
2136 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET;
2138 case MLX5_RXQ_ERR_STATE_NEED_RESET:
2140 sm.queue_id = rxq->idx;
2141 sm.state = IBV_WQS_RESET;
2142 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), &sm))
2144 if (rxq_ctrl->dump_file_n <
2145 rxq_ctrl->priv->config.max_dump_files_num) {
2146 MKSTR(err_str, "Unexpected CQE error syndrome "
2147 "0x%02x CQN = %u RQN = %u wqe_counter = %u"
2148 " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome,
2149 rxq->cqn, rxq_ctrl->wqn,
2150 rte_be_to_cpu_16(u.err_cqe->wqe_counter),
2151 rxq->rq_ci << rxq->sges_n, rxq->cq_ci);
2152 MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u",
2153 rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc());
2154 mlx5_dump_debug_information(name, NULL, err_str, 0);
2155 mlx5_dump_debug_information(name, "MLX5 Error CQ:",
2156 (const void *)((uintptr_t)
2158 sizeof(*u.cqe) * cqe_n);
2159 mlx5_dump_debug_information(name, "MLX5 Error RQ:",
2160 (const void *)((uintptr_t)
2163 rxq_ctrl->dump_file_n++;
2165 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY;
2167 case MLX5_RXQ_ERR_STATE_NEED_READY:
2168 ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci);
2169 if (ret == MLX5_CQE_STATUS_HW_OWN) {
2171 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
2174 * The RQ consumer index must be zeroed while moving
2175 * from RESET state to RDY state.
2177 *rxq->rq_db = rte_cpu_to_be_32(0);
2180 sm.queue_id = rxq->idx;
2181 sm.state = IBV_WQS_RDY;
2182 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv),
2186 const uint16_t q_mask = wqe_n - 1;
2188 struct rte_mbuf **elt;
2190 unsigned int n = wqe_n - (rxq->rq_ci -
2193 for (i = 0; i < (int)n; ++i) {
2194 elt_idx = (rxq->rq_ci + i) & q_mask;
2195 elt = &(*rxq->elts)[elt_idx];
2196 *elt = rte_mbuf_raw_alloc(rxq->mp);
2198 for (i--; i >= 0; --i) {
2199 elt_idx = (rxq->rq_ci +
2203 rte_pktmbuf_free_seg
2210 mlx5_rxq_initialize(rxq);
2211 rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR;
2220 * Get size of the next packet for a given CQE. For compressed CQEs, the
2221 * consumer index is updated only once all packets of the current one have
2225 * Pointer to RX queue.
2229 * Store pointer to mini-CQE if compressed. Otherwise, the pointer is not
2233 * 0 in case of empty CQE, otherwise the packet size in bytes.
2236 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
2237 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe)
2239 struct rxq_zip *zip = &rxq->zip;
2240 uint16_t cqe_n = cqe_cnt + 1;
2246 /* Process compressed data in the CQE and mini arrays. */
2248 volatile struct mlx5_mini_cqe8 (*mc)[8] =
2249 (volatile struct mlx5_mini_cqe8 (*)[8])
2250 (uintptr_t)(&(*rxq->cqes)[zip->ca &
2253 len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt);
2254 *mcqe = &(*mc)[zip->ai & 7];
2255 if ((++zip->ai & 7) == 0) {
2256 /* Invalidate consumed CQEs */
2259 while (idx != end) {
2260 (*rxq->cqes)[idx & cqe_cnt].op_own =
2261 MLX5_CQE_INVALIDATE;
2265 * Increment consumer index to skip the number
2266 * of CQEs consumed. Hardware leaves holes in
2267 * the CQ ring for software use.
2272 if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) {
2273 /* Invalidate the rest */
2277 while (idx != end) {
2278 (*rxq->cqes)[idx & cqe_cnt].op_own =
2279 MLX5_CQE_INVALIDATE;
2282 rxq->cq_ci = zip->cq_ci;
2286 * No compressed data, get next CQE and verify if it is
2293 ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
2294 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
2295 if (unlikely(ret == MLX5_CQE_STATUS_ERR ||
2297 ret = mlx5_rx_err_handle(rxq, 0);
2298 if (ret == MLX5_CQE_STATUS_HW_OWN ||
2306 op_own = cqe->op_own;
2307 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) {
2308 volatile struct mlx5_mini_cqe8 (*mc)[8] =
2309 (volatile struct mlx5_mini_cqe8 (*)[8])
2310 (uintptr_t)(&(*rxq->cqes)
2314 /* Fix endianness. */
2315 zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt);
2317 * Current mini array position is the one
2318 * returned by check_cqe64().
2320 * If completion comprises several mini arrays,
2321 * as a special case the second one is located
2322 * 7 CQEs after the initial CQE instead of 8
2323 * for subsequent ones.
2325 zip->ca = rxq->cq_ci;
2326 zip->na = zip->ca + 7;
2327 /* Compute the next non compressed CQE. */
2329 zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
2330 /* Get packet size to return. */
2331 len = rte_be_to_cpu_32((*mc)[0].byte_cnt);
2334 /* Prefetch all to be invalidated */
2337 while (idx != end) {
2338 rte_prefetch0(&(*rxq->cqes)[(idx) &
2343 len = rte_be_to_cpu_32(cqe->byte_cnt);
2346 if (unlikely(rxq->err_state)) {
2347 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
2348 ++rxq->stats.idropped;
2356 * Translate RX completion flags to offload flags.
2362 * Offload flags (ol_flags) for struct rte_mbuf.
2364 static inline uint32_t
2365 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe)
2367 uint32_t ol_flags = 0;
2368 uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc);
2372 MLX5_CQE_RX_L3_HDR_VALID,
2373 PKT_RX_IP_CKSUM_GOOD) |
2375 MLX5_CQE_RX_L4_HDR_VALID,
2376 PKT_RX_L4_CKSUM_GOOD);
2381 * Fill in mbuf fields from RX completion flags.
2382 * Note that pkt->ol_flags should be initialized outside of this function.
2385 * Pointer to RX queue.
2390 * @param rss_hash_res
2391 * Packet RSS Hash result.
2394 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
2395 volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res)
2397 /* Update packet information. */
2398 pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe);
2399 if (rss_hash_res && rxq->rss_hash) {
2400 pkt->hash.rss = rss_hash_res;
2401 pkt->ol_flags |= PKT_RX_RSS_HASH;
2403 if (rxq->mark && MLX5_FLOW_MARK_IS_VALID(cqe->sop_drop_qpn)) {
2404 pkt->ol_flags |= PKT_RX_FDIR;
2405 if (cqe->sop_drop_qpn !=
2406 rte_cpu_to_be_32(MLX5_FLOW_MARK_DEFAULT)) {
2407 uint32_t mark = cqe->sop_drop_qpn;
2409 pkt->ol_flags |= PKT_RX_FDIR_ID;
2410 pkt->hash.fdir.hi = mlx5_flow_mark_get(mark);
2414 pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);
2415 if (rxq->vlan_strip &&
2416 (cqe->hdr_type_etc & rte_cpu_to_be_16(MLX5_CQE_VLAN_STRIPPED))) {
2417 pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
2418 pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
2420 if (rxq->hw_timestamp) {
2421 pkt->timestamp = rte_be_to_cpu_64(cqe->timestamp);
2422 pkt->ol_flags |= PKT_RX_TIMESTAMP;
2427 * DPDK callback for RX.
2430 * Generic pointer to RX queue structure.
2432 * Array to store received packets.
2434 * Maximum number of packets in array.
2437 * Number of packets successfully received (<= pkts_n).
2440 mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
2442 struct mlx5_rxq_data *rxq = dpdk_rxq;
2443 const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1;
2444 const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1;
2445 const unsigned int sges_n = rxq->sges_n;
2446 struct rte_mbuf *pkt = NULL;
2447 struct rte_mbuf *seg = NULL;
2448 volatile struct mlx5_cqe *cqe =
2449 &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
2451 unsigned int rq_ci = rxq->rq_ci << sges_n;
2452 int len = 0; /* keep its value across iterations. */
2455 unsigned int idx = rq_ci & wqe_cnt;
2456 volatile struct mlx5_wqe_data_seg *wqe =
2457 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
2458 struct rte_mbuf *rep = (*rxq->elts)[idx];
2459 volatile struct mlx5_mini_cqe8 *mcqe = NULL;
2460 uint32_t rss_hash_res;
2468 rep = rte_mbuf_raw_alloc(rxq->mp);
2469 if (unlikely(rep == NULL)) {
2470 ++rxq->stats.rx_nombuf;
2473 * no buffers before we even started,
2474 * bail out silently.
2478 while (pkt != seg) {
2479 assert(pkt != (*rxq->elts)[idx]);
2483 rte_mbuf_raw_free(pkt);
2489 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
2490 len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe);
2492 rte_mbuf_raw_free(rep);
2496 assert(len >= (rxq->crc_present << 2));
2498 /* If compressed, take hash result from mini-CQE. */
2499 rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ?
2501 mcqe->rx_hash_result);
2502 rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
2503 if (rxq->crc_present)
2504 len -= RTE_ETHER_CRC_LEN;
2507 DATA_LEN(rep) = DATA_LEN(seg);
2508 PKT_LEN(rep) = PKT_LEN(seg);
2509 SET_DATA_OFF(rep, DATA_OFF(seg));
2510 PORT(rep) = PORT(seg);
2511 (*rxq->elts)[idx] = rep;
2513 * Fill NIC descriptor with the new buffer. The lkey and size
2514 * of the buffers are already known, only the buffer address
2517 wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
2518 /* If there's only one MR, no need to replace LKey in WQE. */
2519 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
2520 wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
2521 if (len > DATA_LEN(seg)) {
2522 len -= DATA_LEN(seg);
2527 DATA_LEN(seg) = len;
2528 #ifdef MLX5_PMD_SOFT_COUNTERS
2529 /* Increment bytes counter. */
2530 rxq->stats.ibytes += PKT_LEN(pkt);
2532 /* Return packet. */
2537 /* Align consumer index to the next stride. */
2542 if (unlikely((i == 0) && ((rq_ci >> sges_n) == rxq->rq_ci)))
2544 /* Update the consumer index. */
2545 rxq->rq_ci = rq_ci >> sges_n;
2547 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
2549 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
2550 #ifdef MLX5_PMD_SOFT_COUNTERS
2551 /* Increment packets counter. */
2552 rxq->stats.ipackets += i;
2558 mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
2560 struct mlx5_mprq_buf *buf = opaque;
2562 if (rte_atomic16_read(&buf->refcnt) == 1) {
2563 rte_mempool_put(buf->mp, buf);
2564 } else if (rte_atomic16_add_return(&buf->refcnt, -1) == 0) {
2565 rte_atomic16_set(&buf->refcnt, 1);
2566 rte_mempool_put(buf->mp, buf);
2571 mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf)
2573 mlx5_mprq_buf_free_cb(NULL, buf);
2577 mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx)
2579 struct mlx5_mprq_buf *rep = rxq->mprq_repl;
2580 volatile struct mlx5_wqe_data_seg *wqe =
2581 &((volatile struct mlx5_wqe_mprq *)rxq->wqes)[rq_idx].dseg;
2584 assert(rep != NULL);
2585 /* Replace MPRQ buf. */
2586 (*rxq->mprq_bufs)[rq_idx] = rep;
2588 addr = mlx5_mprq_buf_addr(rep);
2589 wqe->addr = rte_cpu_to_be_64((uintptr_t)addr);
2590 /* If there's only one MR, no need to replace LKey in WQE. */
2591 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
2592 wqe->lkey = mlx5_rx_addr2mr(rxq, (uintptr_t)addr);
2593 /* Stash a mbuf for next replacement. */
2594 if (likely(!rte_mempool_get(rxq->mprq_mp, (void **)&rep)))
2595 rxq->mprq_repl = rep;
2597 rxq->mprq_repl = NULL;
2601 * DPDK callback for RX with Multi-Packet RQ support.
2604 * Generic pointer to RX queue structure.
2606 * Array to store received packets.
2608 * Maximum number of packets in array.
2611 * Number of packets successfully received (<= pkts_n).
2614 mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
2616 struct mlx5_rxq_data *rxq = dpdk_rxq;
2617 const unsigned int strd_n = 1 << rxq->strd_num_n;
2618 const unsigned int strd_sz = 1 << rxq->strd_sz_n;
2619 const unsigned int strd_shift =
2620 MLX5_MPRQ_STRIDE_SHIFT_BYTE * rxq->strd_shift_en;
2621 const unsigned int cq_mask = (1 << rxq->cqe_n) - 1;
2622 const unsigned int wq_mask = (1 << rxq->elts_n) - 1;
2623 volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
2625 uint32_t rq_ci = rxq->rq_ci;
2626 uint16_t consumed_strd = rxq->consumed_strd;
2627 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
2629 while (i < pkts_n) {
2630 struct rte_mbuf *pkt;
2638 volatile struct mlx5_mini_cqe8 *mcqe = NULL;
2639 uint32_t rss_hash_res = 0;
2641 if (consumed_strd == strd_n) {
2642 /* Replace WQE only if the buffer is still in use. */
2643 if (rte_atomic16_read(&buf->refcnt) > 1) {
2644 mprq_buf_replace(rxq, rq_ci & wq_mask);
2645 /* Release the old buffer. */
2646 mlx5_mprq_buf_free(buf);
2647 } else if (unlikely(rxq->mprq_repl == NULL)) {
2648 struct mlx5_mprq_buf *rep;
2651 * Currently, the MPRQ mempool is out of buffer
2652 * and doing memcpy regardless of the size of Rx
2653 * packet. Retry allocation to get back to
2656 if (!rte_mempool_get(rxq->mprq_mp,
2658 rxq->mprq_repl = rep;
2660 /* Advance to the next WQE. */
2663 buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
2665 cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
2666 ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe);
2670 strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
2671 MLX5_MPRQ_STRIDE_NUM_SHIFT;
2673 consumed_strd += strd_cnt;
2674 if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
2677 rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res);
2678 strd_idx = rte_be_to_cpu_16(cqe->wqe_counter);
2680 /* mini-CQE for MPRQ doesn't have hash result. */
2681 strd_idx = rte_be_to_cpu_16(mcqe->stride_idx);
2683 assert(strd_idx < strd_n);
2684 assert(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) & wq_mask));
2686 * Currently configured to receive a packet per a stride. But if
2687 * MTU is adjusted through kernel interface, device could
2688 * consume multiple strides without raising an error. In this
2689 * case, the packet should be dropped because it is bigger than
2690 * the max_rx_pkt_len.
2692 if (unlikely(strd_cnt > 1)) {
2693 ++rxq->stats.idropped;
2696 pkt = rte_pktmbuf_alloc(rxq->mp);
2697 if (unlikely(pkt == NULL)) {
2698 ++rxq->stats.rx_nombuf;
2701 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT;
2702 assert((int)len >= (rxq->crc_present << 2));
2703 if (rxq->crc_present)
2704 len -= RTE_ETHER_CRC_LEN;
2705 offset = strd_idx * strd_sz + strd_shift;
2706 addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf), offset);
2707 /* Initialize the offload flag. */
2710 * Memcpy packets to the target mbuf if:
2711 * - The size of packet is smaller than mprq_max_memcpy_len.
2712 * - Out of buffer in the Mempool for Multi-Packet RQ.
2714 if (len <= rxq->mprq_max_memcpy_len || rxq->mprq_repl == NULL) {
2716 * When memcpy'ing packet due to out-of-buffer, the
2717 * packet must be smaller than the target mbuf.
2719 if (unlikely(rte_pktmbuf_tailroom(pkt) < len)) {
2720 rte_pktmbuf_free_seg(pkt);
2721 ++rxq->stats.idropped;
2724 rte_memcpy(rte_pktmbuf_mtod(pkt, void *), addr, len);
2726 rte_iova_t buf_iova;
2727 struct rte_mbuf_ext_shared_info *shinfo;
2728 uint16_t buf_len = strd_cnt * strd_sz;
2730 /* Increment the refcnt of the whole chunk. */
2731 rte_atomic16_add_return(&buf->refcnt, 1);
2732 assert((uint16_t)rte_atomic16_read(&buf->refcnt) <=
2734 addr = RTE_PTR_SUB(addr, RTE_PKTMBUF_HEADROOM);
2736 * MLX5 device doesn't use iova but it is necessary in a
2737 * case where the Rx packet is transmitted via a
2740 buf_iova = rte_mempool_virt2iova(buf) +
2741 RTE_PTR_DIFF(addr, buf);
2742 shinfo = rte_pktmbuf_ext_shinfo_init_helper(addr,
2743 &buf_len, mlx5_mprq_buf_free_cb, buf);
2745 * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when
2746 * attaching the stride to mbuf and more offload flags
2747 * will be added below by calling rxq_cq_to_mbuf().
2748 * Other fields will be overwritten.
2750 rte_pktmbuf_attach_extbuf(pkt, addr, buf_iova, buf_len,
2752 rte_pktmbuf_reset_headroom(pkt);
2753 assert(pkt->ol_flags == EXT_ATTACHED_MBUF);
2755 * Prevent potential overflow due to MTU change through
2758 if (unlikely(rte_pktmbuf_tailroom(pkt) < len)) {
2759 rte_pktmbuf_free_seg(pkt);
2760 ++rxq->stats.idropped;
2764 rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
2766 DATA_LEN(pkt) = len;
2767 PORT(pkt) = rxq->port_id;
2768 #ifdef MLX5_PMD_SOFT_COUNTERS
2769 /* Increment bytes counter. */
2770 rxq->stats.ibytes += PKT_LEN(pkt);
2772 /* Return packet. */
2776 /* Update the consumer indexes. */
2777 rxq->consumed_strd = consumed_strd;
2779 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
2780 if (rq_ci != rxq->rq_ci) {
2783 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
2785 #ifdef MLX5_PMD_SOFT_COUNTERS
2786 /* Increment packets counter. */
2787 rxq->stats.ipackets += i;
2793 * Dummy DPDK callback for TX.
2795 * This function is used to temporarily replace the real callback during
2796 * unsafe control operations on the queue, or in case of error.
2799 * Generic pointer to TX queue structure.
2801 * Packets to transmit.
2803 * Number of packets in array.
2806 * Number of packets successfully transmitted (<= pkts_n).
2809 removed_tx_burst(void *dpdk_txq __rte_unused,
2810 struct rte_mbuf **pkts __rte_unused,
2811 uint16_t pkts_n __rte_unused)
2818 * Dummy DPDK callback for RX.
2820 * This function is used to temporarily replace the real callback during
2821 * unsafe control operations on the queue, or in case of error.
2824 * Generic pointer to RX queue structure.
2826 * Array to store received packets.
2828 * Maximum number of packets in array.
2831 * Number of packets successfully received (<= pkts_n).
2834 removed_rx_burst(void *dpdk_txq __rte_unused,
2835 struct rte_mbuf **pkts __rte_unused,
2836 uint16_t pkts_n __rte_unused)
2843 * Vectorized Rx/Tx routines are not compiled in when required vector
2844 * instructions are not supported on a target architecture. The following null
2845 * stubs are needed for linkage when those are not included outside of this file
2846 * (e.g. mlx5_rxtx_vec_sse.c for x86).
2850 mlx5_tx_burst_raw_vec(void *dpdk_txq __rte_unused,
2851 struct rte_mbuf **pkts __rte_unused,
2852 uint16_t pkts_n __rte_unused)
2858 mlx5_tx_burst_vec(void *dpdk_txq __rte_unused,
2859 struct rte_mbuf **pkts __rte_unused,
2860 uint16_t pkts_n __rte_unused)
2866 mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
2867 struct rte_mbuf **pkts __rte_unused,
2868 uint16_t pkts_n __rte_unused)
2874 mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
2880 mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
2886 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
2892 mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)