1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2021 6WIND S.A.
3 * Copyright 2021 Mellanox Technologies, Ltd
11 #include <rte_mempool.h>
12 #include <rte_prefetch.h>
13 #include <rte_common.h>
14 #include <rte_branch_prediction.h>
15 #include <rte_ether.h>
16 #include <rte_cycles.h>
20 #include <mlx5_common.h>
22 #include "mlx5_autoconf.h"
23 #include "mlx5_defs.h"
25 #include "mlx5_utils.h"
26 #include "mlx5_rxtx.h"
29 #define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
32 * Move QP from error state to running state and initialize indexes.
35 * Pointer to TX queue control structure.
38 * 0 on success, else -1.
41 tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl)
43 struct mlx5_mp_arg_queue_state_modify sm = {
45 .queue_id = txq_ctrl->txq.idx,
48 if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm))
50 txq_ctrl->txq.wqe_ci = 0;
51 txq_ctrl->txq.wqe_pi = 0;
52 txq_ctrl->txq.elts_comp = 0;
56 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */
58 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe)
60 static const uint8_t magic[] = "seen";
64 for (i = 0; i < sizeof(magic); ++i)
65 if (!ret || err_cqe->rsvd1[i] != magic[i]) {
67 err_cqe->rsvd1[i] = magic[i];
76 * Pointer to TX queue structure.
78 * Pointer to the error CQE.
81 * Negative value if queue recovery failed, otherwise
82 * the error completion entry is handled successfully.
85 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq,
86 volatile struct mlx5_err_cqe *err_cqe)
88 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {
89 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1);
90 struct mlx5_txq_ctrl *txq_ctrl =
91 container_of(txq, struct mlx5_txq_ctrl, txq);
92 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter);
93 int seen = check_err_cqe_seen(err_cqe);
95 if (!seen && txq_ctrl->dump_file_n <
96 txq_ctrl->priv->config.max_dump_files_num) {
97 MKSTR(err_str, "Unexpected CQE error syndrome "
98 "0x%02x CQN = %u SQN = %u wqe_counter = %u "
99 "wq_ci = %u cq_ci = %u", err_cqe->syndrome,
100 txq->cqe_s, txq->qp_num_8s >> 8,
101 rte_be_to_cpu_16(err_cqe->wqe_counter),
102 txq->wqe_ci, txq->cq_ci);
103 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u",
104 PORT_ID(txq_ctrl->priv), txq->idx,
105 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc());
106 mlx5_dump_debug_information(name, NULL, err_str, 0);
107 mlx5_dump_debug_information(name, "MLX5 Error CQ:",
108 (const void *)((uintptr_t)
112 mlx5_dump_debug_information(name, "MLX5 Error SQ:",
113 (const void *)((uintptr_t)
117 txq_ctrl->dump_file_n++;
121 * Count errors in WQEs units.
122 * Later it can be improved to count error packets,
123 * for example, by SQ parsing to find how much packets
124 * should be counted for each WQE.
126 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) -
128 if (tx_recover_qp(txq_ctrl)) {
129 /* Recovering failed - retry later on the same WQE. */
132 /* Release all the remaining buffers. */
133 txq_free_elts(txq_ctrl);
139 * Dummy DPDK callback for TX.
141 * This function is used to temporarily replace the real callback during
142 * unsafe control operations on the queue, or in case of error.
145 * Generic pointer to TX queue structure.
147 * Packets to transmit.
149 * Number of packets in array.
152 * Number of packets successfully transmitted (<= pkts_n).
155 removed_tx_burst(void *dpdk_txq __rte_unused,
156 struct rte_mbuf **pkts __rte_unused,
157 uint16_t pkts_n __rte_unused)
164 * Update completion queue consuming index via doorbell
165 * and flush the completed data buffers.
168 * Pointer to TX queue structure.
170 * valid CQE pointer, if not NULL update txq->wqe_pi and flush the buffers.
172 * Configured Tx offloads mask. It is fully defined at
173 * compile time and may be used for optimization.
175 static __rte_always_inline void
176 mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq,
177 volatile struct mlx5_cqe *last_cqe,
178 unsigned int olx __rte_unused)
180 if (likely(last_cqe != NULL)) {
183 txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter);
184 tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m];
185 if (likely(tail != txq->elts_tail)) {
186 mlx5_tx_free_elts(txq, tail, olx);
187 MLX5_ASSERT(tail == txq->elts_tail);
193 * Manage TX completions. This routine checks the CQ for
194 * arrived CQEs, deduces the last accomplished WQE in SQ,
195 * updates SQ producing index and frees all completed mbufs.
198 * Pointer to TX queue structure.
200 * Configured Tx offloads mask. It is fully defined at
201 * compile time and may be used for optimization.
203 * NOTE: not inlined intentionally, it makes tx_burst
204 * routine smaller, simple and faster - from experiments.
207 mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq,
208 unsigned int olx __rte_unused)
210 unsigned int count = MLX5_TX_COMP_MAX_CQE;
211 volatile struct mlx5_cqe *last_cqe = NULL;
212 bool ring_doorbell = false;
216 volatile struct mlx5_cqe *cqe;
218 cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
219 ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
220 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
221 if (likely(ret != MLX5_CQE_STATUS_ERR)) {
222 /* No new CQEs in completion queue. */
223 MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN);
227 * Some error occurred, try to restart.
228 * We have no barrier after WQE related Doorbell
229 * written, make sure all writes are completed
230 * here, before we might perform SQ reset.
233 ret = mlx5_tx_error_cqe_handle
234 (txq, (volatile struct mlx5_err_cqe *)cqe);
235 if (unlikely(ret < 0)) {
237 * Some error occurred on queue error
238 * handling, we do not advance the index
239 * here, allowing to retry on next call.
244 * We are going to fetch all entries with
245 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status.
246 * The send queue is supposed to be empty.
248 ring_doorbell = true;
250 txq->cq_pi = txq->cq_ci;
254 /* Normal transmit completion. */
255 MLX5_ASSERT(txq->cq_ci != txq->cq_pi);
256 #ifdef RTE_LIBRTE_MLX5_DEBUG
257 MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
260 ring_doorbell = true;
264 * We have to restrict the amount of processed CQEs
265 * in one tx_burst routine call. The CQ may be large
266 * and many CQEs may be updated by the NIC in one
267 * transaction. Buffers freeing is time consuming,
268 * multiple iterations may introduce significant latency.
270 if (likely(--count == 0))
273 if (likely(ring_doorbell)) {
274 /* Ring doorbell to notify hardware. */
275 rte_compiler_barrier();
276 *txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
277 mlx5_tx_comp_flush(txq, last_cqe, olx);
282 * DPDK callback to check the status of a Tx descriptor.
287 * The index of the descriptor in the ring.
290 * The status of the Tx descriptor.
293 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
295 struct mlx5_txq_data *__rte_restrict txq = tx_queue;
298 mlx5_tx_handle_completion(txq, 0);
299 used = txq->elts_head - txq->elts_tail;
301 return RTE_ETH_TX_DESC_FULL;
302 return RTE_ETH_TX_DESC_DONE;
306 * Array of declared and compiled Tx burst function and corresponding
307 * supported offloads set. The array is used to select the Tx burst
308 * function for specified offloads set at Tx queue configuration time.
314 MLX5_TXOFF_INFO(full_empw,
315 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
316 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
317 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
318 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
320 MLX5_TXOFF_INFO(none_empw,
321 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
323 MLX5_TXOFF_INFO(md_empw,
324 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
326 MLX5_TXOFF_INFO(mt_empw,
327 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
328 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
330 MLX5_TXOFF_INFO(mtsc_empw,
331 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
332 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
333 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
335 MLX5_TXOFF_INFO(mti_empw,
336 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
337 MLX5_TXOFF_CONFIG_INLINE |
338 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
340 MLX5_TXOFF_INFO(mtv_empw,
341 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
342 MLX5_TXOFF_CONFIG_VLAN |
343 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
345 MLX5_TXOFF_INFO(mtiv_empw,
346 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
347 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
348 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
350 MLX5_TXOFF_INFO(sc_empw,
351 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
352 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
354 MLX5_TXOFF_INFO(sci_empw,
355 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
356 MLX5_TXOFF_CONFIG_INLINE |
357 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
359 MLX5_TXOFF_INFO(scv_empw,
360 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
361 MLX5_TXOFF_CONFIG_VLAN |
362 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
364 MLX5_TXOFF_INFO(sciv_empw,
365 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
366 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
367 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
369 MLX5_TXOFF_INFO(i_empw,
370 MLX5_TXOFF_CONFIG_INLINE |
371 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
373 MLX5_TXOFF_INFO(v_empw,
374 MLX5_TXOFF_CONFIG_VLAN |
375 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
377 MLX5_TXOFF_INFO(iv_empw,
378 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
379 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
381 MLX5_TXOFF_INFO(full_ts_nompw,
382 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP)
384 MLX5_TXOFF_INFO(full_ts_nompwi,
385 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
386 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
387 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
388 MLX5_TXOFF_CONFIG_TXPP)
390 MLX5_TXOFF_INFO(full_ts,
391 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP |
392 MLX5_TXOFF_CONFIG_EMPW)
394 MLX5_TXOFF_INFO(full_ts_noi,
395 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
396 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
397 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
398 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
400 MLX5_TXOFF_INFO(none_ts,
401 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP |
402 MLX5_TXOFF_CONFIG_EMPW)
404 MLX5_TXOFF_INFO(mdi_ts,
405 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
406 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
408 MLX5_TXOFF_INFO(mti_ts,
409 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
410 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
411 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
413 MLX5_TXOFF_INFO(mtiv_ts,
414 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
415 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
416 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP |
417 MLX5_TXOFF_CONFIG_EMPW)
419 MLX5_TXOFF_INFO(full,
420 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
421 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
422 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
423 MLX5_TXOFF_CONFIG_METADATA)
425 MLX5_TXOFF_INFO(none,
426 MLX5_TXOFF_CONFIG_NONE)
429 MLX5_TXOFF_CONFIG_METADATA)
432 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
433 MLX5_TXOFF_CONFIG_METADATA)
435 MLX5_TXOFF_INFO(mtsc,
436 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
437 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
438 MLX5_TXOFF_CONFIG_METADATA)
441 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
442 MLX5_TXOFF_CONFIG_INLINE |
443 MLX5_TXOFF_CONFIG_METADATA)
446 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
447 MLX5_TXOFF_CONFIG_VLAN |
448 MLX5_TXOFF_CONFIG_METADATA)
450 MLX5_TXOFF_INFO(mtiv,
451 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
452 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
453 MLX5_TXOFF_CONFIG_METADATA)
456 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
457 MLX5_TXOFF_CONFIG_METADATA)
460 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
461 MLX5_TXOFF_CONFIG_INLINE |
462 MLX5_TXOFF_CONFIG_METADATA)
465 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
466 MLX5_TXOFF_CONFIG_VLAN |
467 MLX5_TXOFF_CONFIG_METADATA)
469 MLX5_TXOFF_INFO(sciv,
470 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
471 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
472 MLX5_TXOFF_CONFIG_METADATA)
475 MLX5_TXOFF_CONFIG_INLINE |
476 MLX5_TXOFF_CONFIG_METADATA)
479 MLX5_TXOFF_CONFIG_VLAN |
480 MLX5_TXOFF_CONFIG_METADATA)
483 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
484 MLX5_TXOFF_CONFIG_METADATA)
486 MLX5_TXOFF_INFO(none_mpw,
487 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW |
488 MLX5_TXOFF_CONFIG_MPW)
490 MLX5_TXOFF_INFO(mci_mpw,
491 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
492 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
493 MLX5_TXOFF_CONFIG_MPW)
495 MLX5_TXOFF_INFO(mc_mpw,
496 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
497 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW)
499 MLX5_TXOFF_INFO(i_mpw,
500 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
501 MLX5_TXOFF_CONFIG_MPW)
505 * Configure the Tx function to use. The routine checks configured
506 * Tx offloads for the device and selects appropriate Tx burst routine.
507 * There are multiple Tx burst routines compiled from the same template
508 * in the most optimal way for the dedicated Tx offloads set.
511 * Pointer to private data structure.
514 * Pointer to selected Tx burst function.
517 mlx5_select_tx_function(struct rte_eth_dev *dev)
519 struct mlx5_priv *priv = dev->data->dev_private;
520 struct mlx5_dev_config *config = &priv->config;
521 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
522 unsigned int diff = 0, olx = 0, i, m;
525 if (tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) {
526 /* We should support Multi-Segment Packets. */
527 olx |= MLX5_TXOFF_CONFIG_MULTI;
529 if (tx_offloads & (RTE_ETH_TX_OFFLOAD_TCP_TSO |
530 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
531 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO |
532 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO |
533 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO)) {
534 /* We should support TCP Send Offload. */
535 olx |= MLX5_TXOFF_CONFIG_TSO;
537 if (tx_offloads & (RTE_ETH_TX_OFFLOAD_IP_TNL_TSO |
538 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO |
539 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
540 /* We should support Software Parser for Tunnels. */
541 olx |= MLX5_TXOFF_CONFIG_SWP;
543 if (tx_offloads & (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
544 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
545 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
546 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
547 /* We should support IP/TCP/UDP Checksums. */
548 olx |= MLX5_TXOFF_CONFIG_CSUM;
550 if (tx_offloads & RTE_ETH_TX_OFFLOAD_VLAN_INSERT) {
551 /* We should support VLAN insertion. */
552 olx |= MLX5_TXOFF_CONFIG_VLAN;
554 if (tx_offloads & RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP &&
555 rte_mbuf_dynflag_lookup
556 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 &&
557 rte_mbuf_dynfield_lookup
558 (RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) {
559 /* Offload configured, dynamic entities registered. */
560 olx |= MLX5_TXOFF_CONFIG_TXPP;
562 if (priv->txqs_n && (*priv->txqs)[0]) {
563 struct mlx5_txq_data *txd = (*priv->txqs)[0];
565 if (txd->inlen_send) {
567 * Check the data inline requirements. Data inline
568 * is enabled on per device basis, we can check
569 * the first Tx queue only.
571 * If device does not support VLAN insertion in WQE
572 * and some queues are requested to perform VLAN
573 * insertion offload than inline must be enabled.
575 olx |= MLX5_TXOFF_CONFIG_INLINE;
578 if (config->mps == MLX5_MPW_ENHANCED &&
579 config->txq_inline_min <= 0) {
581 * The NIC supports Enhanced Multi-Packet Write
582 * and does not require minimal inline data.
584 olx |= MLX5_TXOFF_CONFIG_EMPW;
586 if (rte_flow_dynf_metadata_avail()) {
587 /* We should support Flow metadata. */
588 olx |= MLX5_TXOFF_CONFIG_METADATA;
590 if (config->mps == MLX5_MPW) {
592 * The NIC supports Legacy Multi-Packet Write.
593 * The MLX5_TXOFF_CONFIG_MPW controls the descriptor building
594 * method in combination with MLX5_TXOFF_CONFIG_EMPW.
596 if (!(olx & (MLX5_TXOFF_CONFIG_TSO |
597 MLX5_TXOFF_CONFIG_SWP |
598 MLX5_TXOFF_CONFIG_VLAN |
599 MLX5_TXOFF_CONFIG_METADATA)))
600 olx |= MLX5_TXOFF_CONFIG_EMPW |
601 MLX5_TXOFF_CONFIG_MPW;
604 * Scan the routines table to find the minimal
605 * satisfying routine with requested offloads.
607 m = RTE_DIM(txoff_func);
608 for (i = 0; i < RTE_DIM(txoff_func); i++) {
611 tmp = txoff_func[i].olx;
613 /* Meets requested offloads exactly.*/
617 if ((tmp & olx) != olx) {
618 /* Does not meet requested offloads at all. */
621 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW)
622 /* Do not enable legacy MPW if not configured. */
624 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
625 /* Do not enable eMPW if not configured. */
627 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
628 /* Do not enable inlining if not configured. */
630 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP)
631 /* Do not enable scheduling if not configured. */
634 * Some routine meets the requirements.
635 * Check whether it has minimal amount
636 * of not requested offloads.
638 tmp = __builtin_popcountl(tmp & ~olx);
639 if (m >= RTE_DIM(txoff_func) || tmp < diff) {
640 /* First or better match, save and continue. */
646 tmp = txoff_func[i].olx ^ txoff_func[m].olx;
647 if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
648 __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
649 /* Lighter not requested offload. */
654 if (m >= RTE_DIM(txoff_func)) {
655 DRV_LOG(DEBUG, "port %u has no selected Tx function"
656 " for requested offloads %04X",
657 dev->data->port_id, olx);
660 DRV_LOG(DEBUG, "port %u has selected Tx function"
661 " supporting offloads %04X/%04X",
662 dev->data->port_id, olx, txoff_func[m].olx);
663 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
664 DRV_LOG(DEBUG, "\tMULTI (multi segment)");
665 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
666 DRV_LOG(DEBUG, "\tTSO (TCP send offload)");
667 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
668 DRV_LOG(DEBUG, "\tSWP (software parser)");
669 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
670 DRV_LOG(DEBUG, "\tCSUM (checksum offload)");
671 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
672 DRV_LOG(DEBUG, "\tINLIN (inline data)");
673 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
674 DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
675 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
676 DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
677 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP)
678 DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)");
679 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) {
680 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW)
681 DRV_LOG(DEBUG, "\tMPW (Legacy MPW)");
683 DRV_LOG(DEBUG, "\tEMPW (Enhanced MPW)");
685 return txoff_func[m].func;
689 * DPDK callback to get the TX queue information.
692 * Pointer to the device structure.
695 * Tx queue identificator.
698 * Pointer to the TX queue information structure.
704 mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
705 struct rte_eth_txq_info *qinfo)
707 struct mlx5_priv *priv = dev->data->dev_private;
708 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
709 struct mlx5_txq_ctrl *txq_ctrl =
710 container_of(txq, struct mlx5_txq_ctrl, txq);
714 qinfo->nb_desc = txq->elts_s;
715 qinfo->conf.tx_thresh.pthresh = 0;
716 qinfo->conf.tx_thresh.hthresh = 0;
717 qinfo->conf.tx_thresh.wthresh = 0;
718 qinfo->conf.tx_rs_thresh = 0;
719 qinfo->conf.tx_free_thresh = 0;
720 qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1;
721 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
725 * DPDK callback to get the TX packet burst mode information.
728 * Pointer to the device structure.
731 * Tx queue identification.
734 * Pointer to the burts mode information.
737 * 0 as success, -EINVAL as failure.
740 mlx5_tx_burst_mode_get(struct rte_eth_dev *dev,
741 uint16_t tx_queue_id,
742 struct rte_eth_burst_mode *mode)
744 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
745 struct mlx5_priv *priv = dev->data->dev_private;
746 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
749 for (i = 0; i < RTE_DIM(txoff_func); i++) {
750 if (pkt_burst == txoff_func[i].func) {
751 olx = txoff_func[i].olx;
752 snprintf(mode->info, sizeof(mode->info),
753 "%s%s%s%s%s%s%s%s%s%s",
754 (olx & MLX5_TXOFF_CONFIG_EMPW) ?
755 ((olx & MLX5_TXOFF_CONFIG_MPW) ?
756 "Legacy MPW" : "Enhanced MPW") : "No MPW",
757 (olx & MLX5_TXOFF_CONFIG_MULTI) ?
759 (olx & MLX5_TXOFF_CONFIG_TSO) ?
761 (olx & MLX5_TXOFF_CONFIG_SWP) ?
763 (olx & MLX5_TXOFF_CONFIG_CSUM) ?
765 (olx & MLX5_TXOFF_CONFIG_INLINE) ?
767 (olx & MLX5_TXOFF_CONFIG_VLAN) ?
769 (olx & MLX5_TXOFF_CONFIG_METADATA) ?
771 (olx & MLX5_TXOFF_CONFIG_TXPP) ?
773 (txq && txq->fast_free) ?
774 " + Fast Free" : "");