1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2015 6WIND S.A.
3 * Copyright 2015-2019 Mellanox Technologies, Ltd
11 #include <rte_mempool.h>
12 #include <rte_prefetch.h>
13 #include <rte_common.h>
14 #include <rte_branch_prediction.h>
15 #include <rte_ether.h>
16 #include <rte_cycles.h>
20 #include <mlx5_common.h>
22 #include "mlx5_autoconf.h"
23 #include "mlx5_defs.h"
26 #include "mlx5_utils.h"
27 #include "mlx5_rxtx.h"
31 #define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
34 static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
35 static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
36 static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
38 sizeof(rte_v128u32_t)),
39 "invalid Ethernet Segment data size");
40 static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
42 sizeof(struct rte_vlan_hdr) +
43 2 * RTE_ETHER_ADDR_LEN),
44 "invalid Ethernet Segment data size");
45 static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
47 sizeof(rte_v128u32_t)),
48 "invalid Ethernet Segment data size");
49 static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
51 sizeof(struct rte_vlan_hdr) +
52 2 * RTE_ETHER_ADDR_LEN),
53 "invalid Ethernet Segment data size");
54 static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
56 sizeof(rte_v128u32_t)),
57 "invalid Ethernet Segment data size");
58 static_assert(MLX5_ESEG_MIN_INLINE_SIZE ==
60 sizeof(struct rte_vlan_hdr) +
61 2 * RTE_ETHER_ADDR_LEN),
62 "invalid Ethernet Segment data size");
63 static_assert(MLX5_DSEG_MIN_INLINE_SIZE ==
64 (2 * RTE_ETHER_ADDR_LEN),
65 "invalid Data Segment data size");
66 static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
67 static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
68 static_assert((sizeof(struct rte_vlan_hdr) +
69 sizeof(struct rte_ether_hdr)) ==
70 MLX5_ESEG_MIN_INLINE_SIZE,
71 "invalid min inline data size");
72 static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <=
73 MLX5_DSEG_MAX, "invalid WQE max size");
74 static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE,
75 "invalid WQE Control Segment size");
76 static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE,
77 "invalid WQE Ethernet Segment size");
78 static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE,
79 "invalid WQE Data Segment size");
80 static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE,
83 uint32_t mlx5_ptype_table[] __rte_cache_aligned = {
84 [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */
87 uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned;
88 uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
90 uint64_t rte_net_mlx5_dynf_inline_mask;
93 * Build a table to translate Rx completion flags to packet type.
95 * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
98 mlx5_set_ptype_table(void)
101 uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table;
103 /* Last entry must not be overwritten, reserved for errored packet. */
104 for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i)
105 (*p)[i] = RTE_PTYPE_UNKNOWN;
107 * The index to the array should have:
108 * bit[1:0] = l3_hdr_type
109 * bit[4:2] = l4_hdr_type
112 * bit[7] = outer_l3_type
115 (*p)[0x00] = RTE_PTYPE_L2_ETHER;
117 (*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
118 RTE_PTYPE_L4_NONFRAG;
119 (*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
120 RTE_PTYPE_L4_NONFRAG;
122 (*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
124 (*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
127 (*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
129 (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
131 (*p)[0x0d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
133 (*p)[0x0e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
135 (*p)[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
137 (*p)[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
140 (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
142 (*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
144 /* Repeat with outer_l3_type being set. Just in case. */
145 (*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
146 RTE_PTYPE_L4_NONFRAG;
147 (*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
148 RTE_PTYPE_L4_NONFRAG;
149 (*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
151 (*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
153 (*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
155 (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
157 (*p)[0x8d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
159 (*p)[0x8e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
161 (*p)[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
163 (*p)[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
165 (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
167 (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
170 (*p)[0x40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
171 (*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
172 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
173 RTE_PTYPE_INNER_L4_NONFRAG;
174 (*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
175 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
176 RTE_PTYPE_INNER_L4_NONFRAG;
177 (*p)[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
178 (*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
179 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
180 RTE_PTYPE_INNER_L4_NONFRAG;
181 (*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
182 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
183 RTE_PTYPE_INNER_L4_NONFRAG;
184 /* Tunneled - Fragmented */
185 (*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
186 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
187 RTE_PTYPE_INNER_L4_FRAG;
188 (*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
189 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
190 RTE_PTYPE_INNER_L4_FRAG;
191 (*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
192 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
193 RTE_PTYPE_INNER_L4_FRAG;
194 (*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
195 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
196 RTE_PTYPE_INNER_L4_FRAG;
198 (*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
199 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
200 RTE_PTYPE_INNER_L4_TCP;
201 (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
202 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
203 RTE_PTYPE_INNER_L4_TCP;
204 (*p)[0x4d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
205 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
206 RTE_PTYPE_INNER_L4_TCP;
207 (*p)[0x4e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
208 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
209 RTE_PTYPE_INNER_L4_TCP;
210 (*p)[0x51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
211 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
212 RTE_PTYPE_INNER_L4_TCP;
213 (*p)[0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
214 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
215 RTE_PTYPE_INNER_L4_TCP;
216 (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
217 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
218 RTE_PTYPE_INNER_L4_TCP;
219 (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
220 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
221 RTE_PTYPE_INNER_L4_TCP;
222 (*p)[0xcd] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
223 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
224 RTE_PTYPE_INNER_L4_TCP;
225 (*p)[0xce] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
226 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
227 RTE_PTYPE_INNER_L4_TCP;
228 (*p)[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
229 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
230 RTE_PTYPE_INNER_L4_TCP;
231 (*p)[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
232 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
233 RTE_PTYPE_INNER_L4_TCP;
235 (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
236 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
237 RTE_PTYPE_INNER_L4_UDP;
238 (*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
239 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
240 RTE_PTYPE_INNER_L4_UDP;
241 (*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
242 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
243 RTE_PTYPE_INNER_L4_UDP;
244 (*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
245 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
246 RTE_PTYPE_INNER_L4_UDP;
250 * Build a table to translate packet to checksum type of Verbs.
253 mlx5_set_cksum_table(void)
259 * The index should have:
260 * bit[0] = PKT_TX_TCP_SEG
261 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
262 * bit[4] = PKT_TX_IP_CKSUM
263 * bit[8] = PKT_TX_OUTER_IP_CKSUM
266 for (i = 0; i < RTE_DIM(mlx5_cksum_table); ++i) {
269 /* Tunneled packet. */
270 if (i & (1 << 8)) /* Outer IP. */
271 v |= MLX5_ETH_WQE_L3_CSUM;
272 if (i & (1 << 4)) /* Inner IP. */
273 v |= MLX5_ETH_WQE_L3_INNER_CSUM;
274 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
275 v |= MLX5_ETH_WQE_L4_INNER_CSUM;
278 if (i & (1 << 4)) /* IP. */
279 v |= MLX5_ETH_WQE_L3_CSUM;
280 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
281 v |= MLX5_ETH_WQE_L4_CSUM;
283 mlx5_cksum_table[i] = v;
288 * Build a table to translate packet type of mbuf to SWP type of Verbs.
291 mlx5_set_swp_types_table(void)
297 * The index should have:
298 * bit[0:1] = PKT_TX_L4_MASK
299 * bit[4] = PKT_TX_IPV6
300 * bit[8] = PKT_TX_OUTER_IPV6
301 * bit[9] = PKT_TX_OUTER_UDP
303 for (i = 0; i < RTE_DIM(mlx5_swp_types_table); ++i) {
306 v |= MLX5_ETH_WQE_L3_OUTER_IPV6;
308 v |= MLX5_ETH_WQE_L4_OUTER_UDP;
310 v |= MLX5_ETH_WQE_L3_INNER_IPV6;
311 if ((i & 3) == (PKT_TX_UDP_CKSUM >> 52))
312 v |= MLX5_ETH_WQE_L4_INNER_UDP;
313 mlx5_swp_types_table[i] = v;
317 #define MLX5_SYSTEM_LOG_DIR "/var/log"
319 * Dump debug information to log file.
324 * If not NULL this string is printed as a header to the output
325 * and the output will be in hexadecimal view.
327 * This is the buffer address to print out.
329 * The number of bytes to dump out.
332 mlx5_dump_debug_information(const char *fname, const char *hex_title,
333 const void *buf, unsigned int hex_len)
337 MKSTR(path, "%s/%s", MLX5_SYSTEM_LOG_DIR, fname);
338 fd = fopen(path, "a+");
340 DRV_LOG(WARNING, "cannot open %s for debug dump", path);
341 MKSTR(path2, "./%s", fname);
342 fd = fopen(path2, "a+");
344 DRV_LOG(ERR, "cannot open %s for debug dump", path2);
347 DRV_LOG(INFO, "New debug dump in file %s", path2);
349 DRV_LOG(INFO, "New debug dump in file %s", path);
352 rte_hexdump(fd, hex_title, buf, hex_len);
354 fprintf(fd, "%s", (const char *)buf);
355 fprintf(fd, "\n\n\n");
360 * Move QP from error state to running state and initialize indexes.
363 * Pointer to TX queue control structure.
366 * 0 on success, else -1.
369 tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl)
371 struct mlx5_mp_arg_queue_state_modify sm = {
373 .queue_id = txq_ctrl->txq.idx,
376 if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm))
378 txq_ctrl->txq.wqe_ci = 0;
379 txq_ctrl->txq.wqe_pi = 0;
380 txq_ctrl->txq.elts_comp = 0;
384 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */
386 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe)
388 static const uint8_t magic[] = "seen";
392 for (i = 0; i < sizeof(magic); ++i)
393 if (!ret || err_cqe->rsvd1[i] != magic[i]) {
395 err_cqe->rsvd1[i] = magic[i];
404 * Pointer to TX queue structure.
406 * Pointer to the error CQE.
409 * Negative value if queue recovery failed, otherwise
410 * the error completion entry is handled successfully.
413 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq,
414 volatile struct mlx5_err_cqe *err_cqe)
416 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {
417 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1);
418 struct mlx5_txq_ctrl *txq_ctrl =
419 container_of(txq, struct mlx5_txq_ctrl, txq);
420 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter);
421 int seen = check_err_cqe_seen(err_cqe);
423 if (!seen && txq_ctrl->dump_file_n <
424 txq_ctrl->priv->config.max_dump_files_num) {
425 MKSTR(err_str, "Unexpected CQE error syndrome "
426 "0x%02x CQN = %u SQN = %u wqe_counter = %u "
427 "wq_ci = %u cq_ci = %u", err_cqe->syndrome,
428 txq->cqe_s, txq->qp_num_8s >> 8,
429 rte_be_to_cpu_16(err_cqe->wqe_counter),
430 txq->wqe_ci, txq->cq_ci);
431 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u",
432 PORT_ID(txq_ctrl->priv), txq->idx,
433 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc());
434 mlx5_dump_debug_information(name, NULL, err_str, 0);
435 mlx5_dump_debug_information(name, "MLX5 Error CQ:",
436 (const void *)((uintptr_t)
440 mlx5_dump_debug_information(name, "MLX5 Error SQ:",
441 (const void *)((uintptr_t)
445 txq_ctrl->dump_file_n++;
449 * Count errors in WQEs units.
450 * Later it can be improved to count error packets,
451 * for example, by SQ parsing to find how much packets
452 * should be counted for each WQE.
454 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) -
456 if (tx_recover_qp(txq_ctrl)) {
457 /* Recovering failed - retry later on the same WQE. */
460 /* Release all the remaining buffers. */
461 txq_free_elts(txq_ctrl);
467 * Modify a Verbs/DevX queue state.
468 * This must be called from the primary process.
471 * Pointer to Ethernet device.
473 * State modify request parameters.
476 * 0 in case of success else non-zero value and rte_errno is set.
479 mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
480 const struct mlx5_mp_arg_queue_state_modify *sm)
483 struct mlx5_priv *priv = dev->data->dev_private;
486 struct mlx5_rxq_data *rxq = (*priv->rxqs)[sm->queue_id];
487 struct mlx5_rxq_ctrl *rxq_ctrl =
488 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
490 ret = priv->obj_ops.rxq_obj_modify(rxq_ctrl->obj, sm->state);
492 DRV_LOG(ERR, "Cannot change Rx WQ state to %u - %s",
493 sm->state, strerror(errno));
498 struct mlx5_txq_data *txq = (*priv->txqs)[sm->queue_id];
499 struct mlx5_txq_ctrl *txq_ctrl =
500 container_of(txq, struct mlx5_txq_ctrl, txq);
502 ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
503 MLX5_TXQ_MOD_ERR2RDY,
504 (uint8_t)priv->dev_port);
512 * Modify a Verbs queue state.
515 * Pointer to Ethernet device.
517 * State modify request parameters.
520 * 0 in case of success else non-zero value.
523 mlx5_queue_state_modify(struct rte_eth_dev *dev,
524 struct mlx5_mp_arg_queue_state_modify *sm)
526 struct mlx5_priv *priv = dev->data->dev_private;
529 switch (rte_eal_process_type()) {
530 case RTE_PROC_PRIMARY:
531 ret = mlx5_queue_state_modify_primary(dev, sm);
533 case RTE_PROC_SECONDARY:
534 ret = mlx5_mp_req_queue_state_modify(&priv->mp_id, sm);
543 * Dummy DPDK callback for TX.
545 * This function is used to temporarily replace the real callback during
546 * unsafe control operations on the queue, or in case of error.
549 * Generic pointer to TX queue structure.
551 * Packets to transmit.
553 * Number of packets in array.
556 * Number of packets successfully transmitted (<= pkts_n).
559 removed_tx_burst(void *dpdk_txq __rte_unused,
560 struct rte_mbuf **pkts __rte_unused,
561 uint16_t pkts_n __rte_unused)
568 * Update completion queue consuming index via doorbell
569 * and flush the completed data buffers.
572 * Pointer to TX queue structure.
573 * @param valid CQE pointer
574 * if not NULL update txq->wqe_pi and flush the buffers
576 * Configured Tx offloads mask. It is fully defined at
577 * compile time and may be used for optimization.
579 static __rte_always_inline void
580 mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq,
581 volatile struct mlx5_cqe *last_cqe,
582 unsigned int olx __rte_unused)
584 if (likely(last_cqe != NULL)) {
587 txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter);
588 tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m];
589 if (likely(tail != txq->elts_tail)) {
590 mlx5_tx_free_elts(txq, tail, olx);
591 MLX5_ASSERT(tail == txq->elts_tail);
597 * Manage TX completions. This routine checks the CQ for
598 * arrived CQEs, deduces the last accomplished WQE in SQ,
599 * updates SQ producing index and frees all completed mbufs.
602 * Pointer to TX queue structure.
604 * Configured Tx offloads mask. It is fully defined at
605 * compile time and may be used for optimization.
607 * NOTE: not inlined intentionally, it makes tx_burst
608 * routine smaller, simple and faster - from experiments.
611 mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq,
612 unsigned int olx __rte_unused)
614 unsigned int count = MLX5_TX_COMP_MAX_CQE;
615 volatile struct mlx5_cqe *last_cqe = NULL;
616 bool ring_doorbell = false;
620 volatile struct mlx5_cqe *cqe;
622 cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
623 ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
624 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
625 if (likely(ret != MLX5_CQE_STATUS_ERR)) {
626 /* No new CQEs in completion queue. */
627 MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN);
631 * Some error occurred, try to restart.
632 * We have no barrier after WQE related Doorbell
633 * written, make sure all writes are completed
634 * here, before we might perform SQ reset.
637 ret = mlx5_tx_error_cqe_handle
638 (txq, (volatile struct mlx5_err_cqe *)cqe);
639 if (unlikely(ret < 0)) {
641 * Some error occurred on queue error
642 * handling, we do not advance the index
643 * here, allowing to retry on next call.
648 * We are going to fetch all entries with
649 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status.
650 * The send queue is supposed to be empty.
652 ring_doorbell = true;
654 txq->cq_pi = txq->cq_ci;
658 /* Normal transmit completion. */
659 MLX5_ASSERT(txq->cq_ci != txq->cq_pi);
660 #ifdef RTE_LIBRTE_MLX5_DEBUG
661 MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
664 ring_doorbell = true;
668 * We have to restrict the amount of processed CQEs
669 * in one tx_burst routine call. The CQ may be large
670 * and many CQEs may be updated by the NIC in one
671 * transaction. Buffers freeing is time consuming,
672 * multiple iterations may introduce significant
675 if (likely(--count == 0))
678 if (likely(ring_doorbell)) {
679 /* Ring doorbell to notify hardware. */
680 rte_compiler_barrier();
681 *txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
682 mlx5_tx_comp_flush(txq, last_cqe, olx);
687 * DPDK callback to check the status of a tx descriptor.
692 * The index of the descriptor in the ring.
695 * The status of the tx descriptor.
698 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
700 struct mlx5_txq_data *__rte_restrict txq = tx_queue;
703 mlx5_tx_handle_completion(txq, 0);
704 used = txq->elts_head - txq->elts_tail;
706 return RTE_ETH_TX_DESC_FULL;
707 return RTE_ETH_TX_DESC_DONE;
710 /* Generate routines with Enhanced Multi-Packet Write support. */
711 MLX5_TXOFF_DECL(full_empw,
712 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_EMPW)
714 MLX5_TXOFF_DECL(none_empw,
715 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
717 MLX5_TXOFF_DECL(md_empw,
718 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
720 MLX5_TXOFF_DECL(mt_empw,
721 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
722 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
724 MLX5_TXOFF_DECL(mtsc_empw,
725 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
726 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
727 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
729 MLX5_TXOFF_DECL(mti_empw,
730 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
731 MLX5_TXOFF_CONFIG_INLINE |
732 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
734 MLX5_TXOFF_DECL(mtv_empw,
735 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
736 MLX5_TXOFF_CONFIG_VLAN |
737 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
739 MLX5_TXOFF_DECL(mtiv_empw,
740 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
741 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
742 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
744 MLX5_TXOFF_DECL(sc_empw,
745 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
746 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
748 MLX5_TXOFF_DECL(sci_empw,
749 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
750 MLX5_TXOFF_CONFIG_INLINE |
751 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
753 MLX5_TXOFF_DECL(scv_empw,
754 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
755 MLX5_TXOFF_CONFIG_VLAN |
756 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
758 MLX5_TXOFF_DECL(sciv_empw,
759 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
760 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
761 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
763 MLX5_TXOFF_DECL(i_empw,
764 MLX5_TXOFF_CONFIG_INLINE |
765 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
767 MLX5_TXOFF_DECL(v_empw,
768 MLX5_TXOFF_CONFIG_VLAN |
769 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
771 MLX5_TXOFF_DECL(iv_empw,
772 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
773 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
775 /* Generate routines without Enhanced Multi-Packet Write support. */
776 MLX5_TXOFF_DECL(full,
777 MLX5_TXOFF_CONFIG_FULL)
779 MLX5_TXOFF_DECL(none,
780 MLX5_TXOFF_CONFIG_NONE)
783 MLX5_TXOFF_CONFIG_METADATA)
786 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
787 MLX5_TXOFF_CONFIG_METADATA)
789 MLX5_TXOFF_DECL(mtsc,
790 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
791 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
792 MLX5_TXOFF_CONFIG_METADATA)
795 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
796 MLX5_TXOFF_CONFIG_INLINE |
797 MLX5_TXOFF_CONFIG_METADATA)
801 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
802 MLX5_TXOFF_CONFIG_VLAN |
803 MLX5_TXOFF_CONFIG_METADATA)
806 MLX5_TXOFF_DECL(mtiv,
807 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
808 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
809 MLX5_TXOFF_CONFIG_METADATA)
812 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
813 MLX5_TXOFF_CONFIG_METADATA)
816 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
817 MLX5_TXOFF_CONFIG_INLINE |
818 MLX5_TXOFF_CONFIG_METADATA)
822 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
823 MLX5_TXOFF_CONFIG_VLAN |
824 MLX5_TXOFF_CONFIG_METADATA)
827 MLX5_TXOFF_DECL(sciv,
828 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
829 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
830 MLX5_TXOFF_CONFIG_METADATA)
833 MLX5_TXOFF_CONFIG_INLINE |
834 MLX5_TXOFF_CONFIG_METADATA)
837 MLX5_TXOFF_CONFIG_VLAN |
838 MLX5_TXOFF_CONFIG_METADATA)
841 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
842 MLX5_TXOFF_CONFIG_METADATA)
844 /* Generate routines with timestamp scheduling. */
845 MLX5_TXOFF_DECL(full_ts_nompw,
846 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP)
848 MLX5_TXOFF_DECL(full_ts_nompwi,
849 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
850 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
851 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
852 MLX5_TXOFF_CONFIG_TXPP)
854 MLX5_TXOFF_DECL(full_ts,
855 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP |
856 MLX5_TXOFF_CONFIG_EMPW)
858 MLX5_TXOFF_DECL(full_ts_noi,
859 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
860 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
861 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
862 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
864 MLX5_TXOFF_DECL(none_ts,
865 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP |
866 MLX5_TXOFF_CONFIG_EMPW)
868 MLX5_TXOFF_DECL(mdi_ts,
869 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
870 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
872 MLX5_TXOFF_DECL(mti_ts,
873 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
874 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
875 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
877 MLX5_TXOFF_DECL(mtiv_ts,
878 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
879 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
880 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP |
881 MLX5_TXOFF_CONFIG_EMPW)
884 * Generate routines with Legacy Multi-Packet Write support.
885 * This mode is supported by ConnectX-4 Lx only and imposes
886 * offload limitations, not supported:
887 * - ACL/Flows (metadata are becoming meaningless)
888 * - WQE Inline headers
889 * - SRIOV (E-Switch offloads)
891 * - tunnel encapsulation/decapsulation
894 MLX5_TXOFF_DECL(none_mpw,
895 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW |
896 MLX5_TXOFF_CONFIG_MPW)
898 MLX5_TXOFF_DECL(mci_mpw,
899 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
900 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
901 MLX5_TXOFF_CONFIG_MPW)
903 MLX5_TXOFF_DECL(mc_mpw,
904 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
905 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW)
907 MLX5_TXOFF_DECL(i_mpw,
908 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
909 MLX5_TXOFF_CONFIG_MPW)
912 * Array of declared and compiled Tx burst function and corresponding
913 * supported offloads set. The array is used to select the Tx burst
914 * function for specified offloads set at Tx queue configuration time.
920 MLX5_TXOFF_INFO(full_empw,
921 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
922 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
923 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
924 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
926 MLX5_TXOFF_INFO(none_empw,
927 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
929 MLX5_TXOFF_INFO(md_empw,
930 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
932 MLX5_TXOFF_INFO(mt_empw,
933 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
934 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
936 MLX5_TXOFF_INFO(mtsc_empw,
937 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
938 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
939 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
941 MLX5_TXOFF_INFO(mti_empw,
942 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
943 MLX5_TXOFF_CONFIG_INLINE |
944 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
946 MLX5_TXOFF_INFO(mtv_empw,
947 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
948 MLX5_TXOFF_CONFIG_VLAN |
949 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
951 MLX5_TXOFF_INFO(mtiv_empw,
952 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
953 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
954 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
956 MLX5_TXOFF_INFO(sc_empw,
957 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
958 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
960 MLX5_TXOFF_INFO(sci_empw,
961 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
962 MLX5_TXOFF_CONFIG_INLINE |
963 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
965 MLX5_TXOFF_INFO(scv_empw,
966 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
967 MLX5_TXOFF_CONFIG_VLAN |
968 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
970 MLX5_TXOFF_INFO(sciv_empw,
971 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
972 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
973 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
975 MLX5_TXOFF_INFO(i_empw,
976 MLX5_TXOFF_CONFIG_INLINE |
977 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
979 MLX5_TXOFF_INFO(v_empw,
980 MLX5_TXOFF_CONFIG_VLAN |
981 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
983 MLX5_TXOFF_INFO(iv_empw,
984 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
985 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
987 MLX5_TXOFF_INFO(full_ts_nompw,
988 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP)
990 MLX5_TXOFF_INFO(full_ts_nompwi,
991 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
992 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
993 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
994 MLX5_TXOFF_CONFIG_TXPP)
996 MLX5_TXOFF_INFO(full_ts,
997 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP |
998 MLX5_TXOFF_CONFIG_EMPW)
1000 MLX5_TXOFF_INFO(full_ts_noi,
1001 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1002 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1003 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
1004 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
1006 MLX5_TXOFF_INFO(none_ts,
1007 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP |
1008 MLX5_TXOFF_CONFIG_EMPW)
1010 MLX5_TXOFF_INFO(mdi_ts,
1011 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
1012 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
1014 MLX5_TXOFF_INFO(mti_ts,
1015 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1016 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
1017 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
1019 MLX5_TXOFF_INFO(mtiv_ts,
1020 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1021 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
1022 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP |
1023 MLX5_TXOFF_CONFIG_EMPW)
1025 MLX5_TXOFF_INFO(full,
1026 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1027 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1028 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
1029 MLX5_TXOFF_CONFIG_METADATA)
1031 MLX5_TXOFF_INFO(none,
1032 MLX5_TXOFF_CONFIG_NONE)
1035 MLX5_TXOFF_CONFIG_METADATA)
1038 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1039 MLX5_TXOFF_CONFIG_METADATA)
1041 MLX5_TXOFF_INFO(mtsc,
1042 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1043 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1044 MLX5_TXOFF_CONFIG_METADATA)
1046 MLX5_TXOFF_INFO(mti,
1047 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1048 MLX5_TXOFF_CONFIG_INLINE |
1049 MLX5_TXOFF_CONFIG_METADATA)
1051 MLX5_TXOFF_INFO(mtv,
1052 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1053 MLX5_TXOFF_CONFIG_VLAN |
1054 MLX5_TXOFF_CONFIG_METADATA)
1056 MLX5_TXOFF_INFO(mtiv,
1057 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
1058 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
1059 MLX5_TXOFF_CONFIG_METADATA)
1062 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1063 MLX5_TXOFF_CONFIG_METADATA)
1065 MLX5_TXOFF_INFO(sci,
1066 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1067 MLX5_TXOFF_CONFIG_INLINE |
1068 MLX5_TXOFF_CONFIG_METADATA)
1070 MLX5_TXOFF_INFO(scv,
1071 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1072 MLX5_TXOFF_CONFIG_VLAN |
1073 MLX5_TXOFF_CONFIG_METADATA)
1075 MLX5_TXOFF_INFO(sciv,
1076 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
1077 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
1078 MLX5_TXOFF_CONFIG_METADATA)
1081 MLX5_TXOFF_CONFIG_INLINE |
1082 MLX5_TXOFF_CONFIG_METADATA)
1085 MLX5_TXOFF_CONFIG_VLAN |
1086 MLX5_TXOFF_CONFIG_METADATA)
1089 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
1090 MLX5_TXOFF_CONFIG_METADATA)
1092 MLX5_TXOFF_INFO(none_mpw,
1093 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW |
1094 MLX5_TXOFF_CONFIG_MPW)
1096 MLX5_TXOFF_INFO(mci_mpw,
1097 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
1098 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
1099 MLX5_TXOFF_CONFIG_MPW)
1101 MLX5_TXOFF_INFO(mc_mpw,
1102 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
1103 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW)
1105 MLX5_TXOFF_INFO(i_mpw,
1106 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
1107 MLX5_TXOFF_CONFIG_MPW)
1111 * Configure the Tx function to use. The routine checks configured
1112 * Tx offloads for the device and selects appropriate Tx burst
1113 * routine. There are multiple Tx burst routines compiled from
1114 * the same template in the most optimal way for the dedicated
1118 * Pointer to private data structure.
1121 * Pointer to selected Tx burst function.
1124 mlx5_select_tx_function(struct rte_eth_dev *dev)
1126 struct mlx5_priv *priv = dev->data->dev_private;
1127 struct mlx5_dev_config *config = &priv->config;
1128 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
1129 unsigned int diff = 0, olx = 0, i, m;
1132 if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
1133 /* We should support Multi-Segment Packets. */
1134 olx |= MLX5_TXOFF_CONFIG_MULTI;
1136 if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
1137 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1138 DEV_TX_OFFLOAD_GRE_TNL_TSO |
1139 DEV_TX_OFFLOAD_IP_TNL_TSO |
1140 DEV_TX_OFFLOAD_UDP_TNL_TSO)) {
1141 /* We should support TCP Send Offload. */
1142 olx |= MLX5_TXOFF_CONFIG_TSO;
1144 if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
1145 DEV_TX_OFFLOAD_UDP_TNL_TSO |
1146 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
1147 /* We should support Software Parser for Tunnels. */
1148 olx |= MLX5_TXOFF_CONFIG_SWP;
1150 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
1151 DEV_TX_OFFLOAD_UDP_CKSUM |
1152 DEV_TX_OFFLOAD_TCP_CKSUM |
1153 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
1154 /* We should support IP/TCP/UDP Checksums. */
1155 olx |= MLX5_TXOFF_CONFIG_CSUM;
1157 if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) {
1158 /* We should support VLAN insertion. */
1159 olx |= MLX5_TXOFF_CONFIG_VLAN;
1161 if (tx_offloads & DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP &&
1162 rte_mbuf_dynflag_lookup
1163 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 &&
1164 rte_mbuf_dynfield_lookup
1165 (RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) {
1166 /* Offload configured, dynamic entities registered. */
1167 olx |= MLX5_TXOFF_CONFIG_TXPP;
1169 if (priv->txqs_n && (*priv->txqs)[0]) {
1170 struct mlx5_txq_data *txd = (*priv->txqs)[0];
1172 if (txd->inlen_send) {
1174 * Check the data inline requirements. Data inline
1175 * is enabled on per device basis, we can check
1176 * the first Tx queue only.
1178 * If device does not support VLAN insertion in WQE
1179 * and some queues are requested to perform VLAN
1180 * insertion offload than inline must be enabled.
1182 olx |= MLX5_TXOFF_CONFIG_INLINE;
1185 if (config->mps == MLX5_MPW_ENHANCED &&
1186 config->txq_inline_min <= 0) {
1188 * The NIC supports Enhanced Multi-Packet Write
1189 * and does not require minimal inline data.
1191 olx |= MLX5_TXOFF_CONFIG_EMPW;
1193 if (rte_flow_dynf_metadata_avail()) {
1194 /* We should support Flow metadata. */
1195 olx |= MLX5_TXOFF_CONFIG_METADATA;
1197 if (config->mps == MLX5_MPW) {
1199 * The NIC supports Legacy Multi-Packet Write.
1200 * The MLX5_TXOFF_CONFIG_MPW controls the
1201 * descriptor building method in combination
1202 * with MLX5_TXOFF_CONFIG_EMPW.
1204 if (!(olx & (MLX5_TXOFF_CONFIG_TSO |
1205 MLX5_TXOFF_CONFIG_SWP |
1206 MLX5_TXOFF_CONFIG_VLAN |
1207 MLX5_TXOFF_CONFIG_METADATA)))
1208 olx |= MLX5_TXOFF_CONFIG_EMPW |
1209 MLX5_TXOFF_CONFIG_MPW;
1212 * Scan the routines table to find the minimal
1213 * satisfying routine with requested offloads.
1215 m = RTE_DIM(txoff_func);
1216 for (i = 0; i < RTE_DIM(txoff_func); i++) {
1219 tmp = txoff_func[i].olx;
1221 /* Meets requested offloads exactly.*/
1225 if ((tmp & olx) != olx) {
1226 /* Does not meet requested offloads at all. */
1229 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW)
1230 /* Do not enable legacy MPW if not configured. */
1232 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
1233 /* Do not enable eMPW if not configured. */
1235 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
1236 /* Do not enable inlining if not configured. */
1238 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP)
1239 /* Do not enable scheduling if not configured. */
1242 * Some routine meets the requirements.
1243 * Check whether it has minimal amount
1244 * of not requested offloads.
1246 tmp = __builtin_popcountl(tmp & ~olx);
1247 if (m >= RTE_DIM(txoff_func) || tmp < diff) {
1248 /* First or better match, save and continue. */
1254 tmp = txoff_func[i].olx ^ txoff_func[m].olx;
1255 if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
1256 __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
1257 /* Lighter not requested offload. */
1262 if (m >= RTE_DIM(txoff_func)) {
1263 DRV_LOG(DEBUG, "port %u has no selected Tx function"
1264 " for requested offloads %04X",
1265 dev->data->port_id, olx);
1268 DRV_LOG(DEBUG, "port %u has selected Tx function"
1269 " supporting offloads %04X/%04X",
1270 dev->data->port_id, olx, txoff_func[m].olx);
1271 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
1272 DRV_LOG(DEBUG, "\tMULTI (multi segment)");
1273 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
1274 DRV_LOG(DEBUG, "\tTSO (TCP send offload)");
1275 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
1276 DRV_LOG(DEBUG, "\tSWP (software parser)");
1277 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
1278 DRV_LOG(DEBUG, "\tCSUM (checksum offload)");
1279 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
1280 DRV_LOG(DEBUG, "\tINLIN (inline data)");
1281 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
1282 DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
1283 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
1284 DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
1285 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP)
1286 DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)");
1287 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) {
1288 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW)
1289 DRV_LOG(DEBUG, "\tMPW (Legacy MPW)");
1291 DRV_LOG(DEBUG, "\tEMPW (Enhanced MPW)");
1293 return txoff_func[m].func;
1297 * DPDK callback to get the TX queue information
1300 * Pointer to the device structure.
1302 * @param tx_queue_id
1303 * Tx queue identificator.
1306 * Pointer to the TX queue information structure.
1313 mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1314 struct rte_eth_txq_info *qinfo)
1316 struct mlx5_priv *priv = dev->data->dev_private;
1317 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
1318 struct mlx5_txq_ctrl *txq_ctrl =
1319 container_of(txq, struct mlx5_txq_ctrl, txq);
1323 qinfo->nb_desc = txq->elts_s;
1324 qinfo->conf.tx_thresh.pthresh = 0;
1325 qinfo->conf.tx_thresh.hthresh = 0;
1326 qinfo->conf.tx_thresh.wthresh = 0;
1327 qinfo->conf.tx_rs_thresh = 0;
1328 qinfo->conf.tx_free_thresh = 0;
1329 qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1;
1330 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
1334 * DPDK callback to get the TX packet burst mode information
1337 * Pointer to the device structure.
1339 * @param tx_queue_id
1340 * Tx queue identificatior.
1343 * Pointer to the burts mode information.
1346 * 0 as success, -EINVAL as failure.
1350 mlx5_tx_burst_mode_get(struct rte_eth_dev *dev,
1351 uint16_t tx_queue_id,
1352 struct rte_eth_burst_mode *mode)
1354 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
1355 struct mlx5_priv *priv = dev->data->dev_private;
1356 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
1357 unsigned int i, olx;
1359 for (i = 0; i < RTE_DIM(txoff_func); i++) {
1360 if (pkt_burst == txoff_func[i].func) {
1361 olx = txoff_func[i].olx;
1362 snprintf(mode->info, sizeof(mode->info),
1363 "%s%s%s%s%s%s%s%s%s%s",
1364 (olx & MLX5_TXOFF_CONFIG_EMPW) ?
1365 ((olx & MLX5_TXOFF_CONFIG_MPW) ?
1366 "Legacy MPW" : "Enhanced MPW") : "No MPW",
1367 (olx & MLX5_TXOFF_CONFIG_MULTI) ?
1369 (olx & MLX5_TXOFF_CONFIG_TSO) ?
1371 (olx & MLX5_TXOFF_CONFIG_SWP) ?
1373 (olx & MLX5_TXOFF_CONFIG_CSUM) ?
1375 (olx & MLX5_TXOFF_CONFIG_INLINE) ?
1377 (olx & MLX5_TXOFF_CONFIG_VLAN) ?
1379 (olx & MLX5_TXOFF_CONFIG_METADATA) ?
1381 (olx & MLX5_TXOFF_CONFIG_TXPP) ?
1383 (txq && txq->fast_free) ?
1384 " + Fast Free" : "");