1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2021 6WIND S.A.
3 * Copyright 2021 Mellanox Technologies, Ltd
11 #include <rte_mempool.h>
12 #include <rte_prefetch.h>
13 #include <rte_common.h>
14 #include <rte_branch_prediction.h>
15 #include <rte_ether.h>
16 #include <rte_cycles.h>
20 #include <mlx5_common.h>
22 #include "mlx5_autoconf.h"
23 #include "mlx5_defs.h"
26 #include "mlx5_utils.h"
27 #include "mlx5_rxtx.h"
30 #define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
33 * Move QP from error state to running state and initialize indexes.
36 * Pointer to TX queue control structure.
39 * 0 on success, else -1.
42 tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl)
44 struct mlx5_mp_arg_queue_state_modify sm = {
46 .queue_id = txq_ctrl->txq.idx,
49 if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm))
51 txq_ctrl->txq.wqe_ci = 0;
52 txq_ctrl->txq.wqe_pi = 0;
53 txq_ctrl->txq.elts_comp = 0;
57 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */
59 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe)
61 static const uint8_t magic[] = "seen";
65 for (i = 0; i < sizeof(magic); ++i)
66 if (!ret || err_cqe->rsvd1[i] != magic[i]) {
68 err_cqe->rsvd1[i] = magic[i];
77 * Pointer to TX queue structure.
79 * Pointer to the error CQE.
82 * Negative value if queue recovery failed, otherwise
83 * the error completion entry is handled successfully.
86 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq,
87 volatile struct mlx5_err_cqe *err_cqe)
89 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {
90 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1);
91 struct mlx5_txq_ctrl *txq_ctrl =
92 container_of(txq, struct mlx5_txq_ctrl, txq);
93 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter);
94 int seen = check_err_cqe_seen(err_cqe);
96 if (!seen && txq_ctrl->dump_file_n <
97 txq_ctrl->priv->config.max_dump_files_num) {
98 MKSTR(err_str, "Unexpected CQE error syndrome "
99 "0x%02x CQN = %u SQN = %u wqe_counter = %u "
100 "wq_ci = %u cq_ci = %u", err_cqe->syndrome,
101 txq->cqe_s, txq->qp_num_8s >> 8,
102 rte_be_to_cpu_16(err_cqe->wqe_counter),
103 txq->wqe_ci, txq->cq_ci);
104 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u",
105 PORT_ID(txq_ctrl->priv), txq->idx,
106 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc());
107 mlx5_dump_debug_information(name, NULL, err_str, 0);
108 mlx5_dump_debug_information(name, "MLX5 Error CQ:",
109 (const void *)((uintptr_t)
113 mlx5_dump_debug_information(name, "MLX5 Error SQ:",
114 (const void *)((uintptr_t)
118 txq_ctrl->dump_file_n++;
122 * Count errors in WQEs units.
123 * Later it can be improved to count error packets,
124 * for example, by SQ parsing to find how much packets
125 * should be counted for each WQE.
127 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) -
129 if (tx_recover_qp(txq_ctrl)) {
130 /* Recovering failed - retry later on the same WQE. */
133 /* Release all the remaining buffers. */
134 txq_free_elts(txq_ctrl);
140 * Dummy DPDK callback for TX.
142 * This function is used to temporarily replace the real callback during
143 * unsafe control operations on the queue, or in case of error.
146 * Generic pointer to TX queue structure.
148 * Packets to transmit.
150 * Number of packets in array.
153 * Number of packets successfully transmitted (<= pkts_n).
156 removed_tx_burst(void *dpdk_txq __rte_unused,
157 struct rte_mbuf **pkts __rte_unused,
158 uint16_t pkts_n __rte_unused)
165 * Update completion queue consuming index via doorbell
166 * and flush the completed data buffers.
169 * Pointer to TX queue structure.
171 * valid CQE pointer, if not NULL update txq->wqe_pi and flush the buffers.
173 * Configured Tx offloads mask. It is fully defined at
174 * compile time and may be used for optimization.
176 static __rte_always_inline void
177 mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq,
178 volatile struct mlx5_cqe *last_cqe,
179 unsigned int olx __rte_unused)
181 if (likely(last_cqe != NULL)) {
184 txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter);
185 tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m];
186 if (likely(tail != txq->elts_tail)) {
187 mlx5_tx_free_elts(txq, tail, olx);
188 MLX5_ASSERT(tail == txq->elts_tail);
194 * Manage TX completions. This routine checks the CQ for
195 * arrived CQEs, deduces the last accomplished WQE in SQ,
196 * updates SQ producing index and frees all completed mbufs.
199 * Pointer to TX queue structure.
201 * Configured Tx offloads mask. It is fully defined at
202 * compile time and may be used for optimization.
204 * NOTE: not inlined intentionally, it makes tx_burst
205 * routine smaller, simple and faster - from experiments.
208 mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq,
209 unsigned int olx __rte_unused)
211 unsigned int count = MLX5_TX_COMP_MAX_CQE;
212 volatile struct mlx5_cqe *last_cqe = NULL;
213 bool ring_doorbell = false;
217 volatile struct mlx5_cqe *cqe;
219 cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
220 ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
221 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
222 if (likely(ret != MLX5_CQE_STATUS_ERR)) {
223 /* No new CQEs in completion queue. */
224 MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN);
228 * Some error occurred, try to restart.
229 * We have no barrier after WQE related Doorbell
230 * written, make sure all writes are completed
231 * here, before we might perform SQ reset.
234 ret = mlx5_tx_error_cqe_handle
235 (txq, (volatile struct mlx5_err_cqe *)cqe);
236 if (unlikely(ret < 0)) {
238 * Some error occurred on queue error
239 * handling, we do not advance the index
240 * here, allowing to retry on next call.
245 * We are going to fetch all entries with
246 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status.
247 * The send queue is supposed to be empty.
249 ring_doorbell = true;
251 txq->cq_pi = txq->cq_ci;
255 /* Normal transmit completion. */
256 MLX5_ASSERT(txq->cq_ci != txq->cq_pi);
257 #ifdef RTE_LIBRTE_MLX5_DEBUG
258 MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
261 ring_doorbell = true;
265 * We have to restrict the amount of processed CQEs
266 * in one tx_burst routine call. The CQ may be large
267 * and many CQEs may be updated by the NIC in one
268 * transaction. Buffers freeing is time consuming,
269 * multiple iterations may introduce significant latency.
271 if (likely(--count == 0))
274 if (likely(ring_doorbell)) {
275 /* Ring doorbell to notify hardware. */
276 rte_compiler_barrier();
277 *txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
278 mlx5_tx_comp_flush(txq, last_cqe, olx);
283 * DPDK callback to check the status of a Tx descriptor.
288 * The index of the descriptor in the ring.
291 * The status of the Tx descriptor.
294 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
296 struct mlx5_txq_data *__rte_restrict txq = tx_queue;
299 mlx5_tx_handle_completion(txq, 0);
300 used = txq->elts_head - txq->elts_tail;
302 return RTE_ETH_TX_DESC_FULL;
303 return RTE_ETH_TX_DESC_DONE;
307 * Array of declared and compiled Tx burst function and corresponding
308 * supported offloads set. The array is used to select the Tx burst
309 * function for specified offloads set at Tx queue configuration time.
315 MLX5_TXOFF_INFO(full_empw,
316 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
317 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
318 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
319 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
321 MLX5_TXOFF_INFO(none_empw,
322 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
324 MLX5_TXOFF_INFO(md_empw,
325 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
327 MLX5_TXOFF_INFO(mt_empw,
328 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
329 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
331 MLX5_TXOFF_INFO(mtsc_empw,
332 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
333 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
334 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
336 MLX5_TXOFF_INFO(mti_empw,
337 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
338 MLX5_TXOFF_CONFIG_INLINE |
339 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
341 MLX5_TXOFF_INFO(mtv_empw,
342 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
343 MLX5_TXOFF_CONFIG_VLAN |
344 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
346 MLX5_TXOFF_INFO(mtiv_empw,
347 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
348 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
349 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
351 MLX5_TXOFF_INFO(sc_empw,
352 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
353 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
355 MLX5_TXOFF_INFO(sci_empw,
356 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
357 MLX5_TXOFF_CONFIG_INLINE |
358 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
360 MLX5_TXOFF_INFO(scv_empw,
361 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
362 MLX5_TXOFF_CONFIG_VLAN |
363 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
365 MLX5_TXOFF_INFO(sciv_empw,
366 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
367 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
368 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
370 MLX5_TXOFF_INFO(i_empw,
371 MLX5_TXOFF_CONFIG_INLINE |
372 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
374 MLX5_TXOFF_INFO(v_empw,
375 MLX5_TXOFF_CONFIG_VLAN |
376 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
378 MLX5_TXOFF_INFO(iv_empw,
379 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
380 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
382 MLX5_TXOFF_INFO(full_ts_nompw,
383 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP)
385 MLX5_TXOFF_INFO(full_ts_nompwi,
386 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
387 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
388 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
389 MLX5_TXOFF_CONFIG_TXPP)
391 MLX5_TXOFF_INFO(full_ts,
392 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP |
393 MLX5_TXOFF_CONFIG_EMPW)
395 MLX5_TXOFF_INFO(full_ts_noi,
396 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
397 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
398 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
399 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
401 MLX5_TXOFF_INFO(none_ts,
402 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP |
403 MLX5_TXOFF_CONFIG_EMPW)
405 MLX5_TXOFF_INFO(mdi_ts,
406 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
407 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
409 MLX5_TXOFF_INFO(mti_ts,
410 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
411 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
412 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
414 MLX5_TXOFF_INFO(mtiv_ts,
415 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
416 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
417 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP |
418 MLX5_TXOFF_CONFIG_EMPW)
420 MLX5_TXOFF_INFO(full,
421 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
422 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
423 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
424 MLX5_TXOFF_CONFIG_METADATA)
426 MLX5_TXOFF_INFO(none,
427 MLX5_TXOFF_CONFIG_NONE)
430 MLX5_TXOFF_CONFIG_METADATA)
433 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
434 MLX5_TXOFF_CONFIG_METADATA)
436 MLX5_TXOFF_INFO(mtsc,
437 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
438 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
439 MLX5_TXOFF_CONFIG_METADATA)
442 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
443 MLX5_TXOFF_CONFIG_INLINE |
444 MLX5_TXOFF_CONFIG_METADATA)
447 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
448 MLX5_TXOFF_CONFIG_VLAN |
449 MLX5_TXOFF_CONFIG_METADATA)
451 MLX5_TXOFF_INFO(mtiv,
452 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
453 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
454 MLX5_TXOFF_CONFIG_METADATA)
457 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
458 MLX5_TXOFF_CONFIG_METADATA)
461 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
462 MLX5_TXOFF_CONFIG_INLINE |
463 MLX5_TXOFF_CONFIG_METADATA)
466 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
467 MLX5_TXOFF_CONFIG_VLAN |
468 MLX5_TXOFF_CONFIG_METADATA)
470 MLX5_TXOFF_INFO(sciv,
471 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
472 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
473 MLX5_TXOFF_CONFIG_METADATA)
476 MLX5_TXOFF_CONFIG_INLINE |
477 MLX5_TXOFF_CONFIG_METADATA)
480 MLX5_TXOFF_CONFIG_VLAN |
481 MLX5_TXOFF_CONFIG_METADATA)
484 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
485 MLX5_TXOFF_CONFIG_METADATA)
487 MLX5_TXOFF_INFO(none_mpw,
488 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW |
489 MLX5_TXOFF_CONFIG_MPW)
491 MLX5_TXOFF_INFO(mci_mpw,
492 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
493 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
494 MLX5_TXOFF_CONFIG_MPW)
496 MLX5_TXOFF_INFO(mc_mpw,
497 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
498 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW)
500 MLX5_TXOFF_INFO(i_mpw,
501 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
502 MLX5_TXOFF_CONFIG_MPW)
506 * Configure the Tx function to use. The routine checks configured
507 * Tx offloads for the device and selects appropriate Tx burst routine.
508 * There are multiple Tx burst routines compiled from the same template
509 * in the most optimal way for the dedicated Tx offloads set.
512 * Pointer to private data structure.
515 * Pointer to selected Tx burst function.
518 mlx5_select_tx_function(struct rte_eth_dev *dev)
520 struct mlx5_priv *priv = dev->data->dev_private;
521 struct mlx5_dev_config *config = &priv->config;
522 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
523 unsigned int diff = 0, olx = 0, i, m;
526 if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
527 /* We should support Multi-Segment Packets. */
528 olx |= MLX5_TXOFF_CONFIG_MULTI;
530 if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
531 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
532 DEV_TX_OFFLOAD_GRE_TNL_TSO |
533 DEV_TX_OFFLOAD_IP_TNL_TSO |
534 DEV_TX_OFFLOAD_UDP_TNL_TSO)) {
535 /* We should support TCP Send Offload. */
536 olx |= MLX5_TXOFF_CONFIG_TSO;
538 if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
539 DEV_TX_OFFLOAD_UDP_TNL_TSO |
540 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
541 /* We should support Software Parser for Tunnels. */
542 olx |= MLX5_TXOFF_CONFIG_SWP;
544 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
545 DEV_TX_OFFLOAD_UDP_CKSUM |
546 DEV_TX_OFFLOAD_TCP_CKSUM |
547 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
548 /* We should support IP/TCP/UDP Checksums. */
549 olx |= MLX5_TXOFF_CONFIG_CSUM;
551 if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) {
552 /* We should support VLAN insertion. */
553 olx |= MLX5_TXOFF_CONFIG_VLAN;
555 if (tx_offloads & DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP &&
556 rte_mbuf_dynflag_lookup
557 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 &&
558 rte_mbuf_dynfield_lookup
559 (RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) {
560 /* Offload configured, dynamic entities registered. */
561 olx |= MLX5_TXOFF_CONFIG_TXPP;
563 if (priv->txqs_n && (*priv->txqs)[0]) {
564 struct mlx5_txq_data *txd = (*priv->txqs)[0];
566 if (txd->inlen_send) {
568 * Check the data inline requirements. Data inline
569 * is enabled on per device basis, we can check
570 * the first Tx queue only.
572 * If device does not support VLAN insertion in WQE
573 * and some queues are requested to perform VLAN
574 * insertion offload than inline must be enabled.
576 olx |= MLX5_TXOFF_CONFIG_INLINE;
579 if (config->mps == MLX5_MPW_ENHANCED &&
580 config->txq_inline_min <= 0) {
582 * The NIC supports Enhanced Multi-Packet Write
583 * and does not require minimal inline data.
585 olx |= MLX5_TXOFF_CONFIG_EMPW;
587 if (rte_flow_dynf_metadata_avail()) {
588 /* We should support Flow metadata. */
589 olx |= MLX5_TXOFF_CONFIG_METADATA;
591 if (config->mps == MLX5_MPW) {
593 * The NIC supports Legacy Multi-Packet Write.
594 * The MLX5_TXOFF_CONFIG_MPW controls the descriptor building
595 * method in combination with MLX5_TXOFF_CONFIG_EMPW.
597 if (!(olx & (MLX5_TXOFF_CONFIG_TSO |
598 MLX5_TXOFF_CONFIG_SWP |
599 MLX5_TXOFF_CONFIG_VLAN |
600 MLX5_TXOFF_CONFIG_METADATA)))
601 olx |= MLX5_TXOFF_CONFIG_EMPW |
602 MLX5_TXOFF_CONFIG_MPW;
605 * Scan the routines table to find the minimal
606 * satisfying routine with requested offloads.
608 m = RTE_DIM(txoff_func);
609 for (i = 0; i < RTE_DIM(txoff_func); i++) {
612 tmp = txoff_func[i].olx;
614 /* Meets requested offloads exactly.*/
618 if ((tmp & olx) != olx) {
619 /* Does not meet requested offloads at all. */
622 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW)
623 /* Do not enable legacy MPW if not configured. */
625 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
626 /* Do not enable eMPW if not configured. */
628 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
629 /* Do not enable inlining if not configured. */
631 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP)
632 /* Do not enable scheduling if not configured. */
635 * Some routine meets the requirements.
636 * Check whether it has minimal amount
637 * of not requested offloads.
639 tmp = __builtin_popcountl(tmp & ~olx);
640 if (m >= RTE_DIM(txoff_func) || tmp < diff) {
641 /* First or better match, save and continue. */
647 tmp = txoff_func[i].olx ^ txoff_func[m].olx;
648 if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
649 __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
650 /* Lighter not requested offload. */
655 if (m >= RTE_DIM(txoff_func)) {
656 DRV_LOG(DEBUG, "port %u has no selected Tx function"
657 " for requested offloads %04X",
658 dev->data->port_id, olx);
661 DRV_LOG(DEBUG, "port %u has selected Tx function"
662 " supporting offloads %04X/%04X",
663 dev->data->port_id, olx, txoff_func[m].olx);
664 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
665 DRV_LOG(DEBUG, "\tMULTI (multi segment)");
666 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
667 DRV_LOG(DEBUG, "\tTSO (TCP send offload)");
668 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
669 DRV_LOG(DEBUG, "\tSWP (software parser)");
670 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
671 DRV_LOG(DEBUG, "\tCSUM (checksum offload)");
672 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
673 DRV_LOG(DEBUG, "\tINLIN (inline data)");
674 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
675 DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
676 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
677 DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
678 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP)
679 DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)");
680 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) {
681 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW)
682 DRV_LOG(DEBUG, "\tMPW (Legacy MPW)");
684 DRV_LOG(DEBUG, "\tEMPW (Enhanced MPW)");
686 return txoff_func[m].func;
690 * DPDK callback to get the TX queue information.
693 * Pointer to the device structure.
696 * Tx queue identificator.
699 * Pointer to the TX queue information structure.
705 mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
706 struct rte_eth_txq_info *qinfo)
708 struct mlx5_priv *priv = dev->data->dev_private;
709 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
710 struct mlx5_txq_ctrl *txq_ctrl =
711 container_of(txq, struct mlx5_txq_ctrl, txq);
715 qinfo->nb_desc = txq->elts_s;
716 qinfo->conf.tx_thresh.pthresh = 0;
717 qinfo->conf.tx_thresh.hthresh = 0;
718 qinfo->conf.tx_thresh.wthresh = 0;
719 qinfo->conf.tx_rs_thresh = 0;
720 qinfo->conf.tx_free_thresh = 0;
721 qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1;
722 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
726 * DPDK callback to get the TX packet burst mode information.
729 * Pointer to the device structure.
732 * Tx queue identificatior.
735 * Pointer to the burts mode information.
738 * 0 as success, -EINVAL as failure.
741 mlx5_tx_burst_mode_get(struct rte_eth_dev *dev,
742 uint16_t tx_queue_id,
743 struct rte_eth_burst_mode *mode)
745 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
746 struct mlx5_priv *priv = dev->data->dev_private;
747 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
750 for (i = 0; i < RTE_DIM(txoff_func); i++) {
751 if (pkt_burst == txoff_func[i].func) {
752 olx = txoff_func[i].olx;
753 snprintf(mode->info, sizeof(mode->info),
754 "%s%s%s%s%s%s%s%s%s%s",
755 (olx & MLX5_TXOFF_CONFIG_EMPW) ?
756 ((olx & MLX5_TXOFF_CONFIG_MPW) ?
757 "Legacy MPW" : "Enhanced MPW") : "No MPW",
758 (olx & MLX5_TXOFF_CONFIG_MULTI) ?
760 (olx & MLX5_TXOFF_CONFIG_TSO) ?
762 (olx & MLX5_TXOFF_CONFIG_SWP) ?
764 (olx & MLX5_TXOFF_CONFIG_CSUM) ?
766 (olx & MLX5_TXOFF_CONFIG_INLINE) ?
768 (olx & MLX5_TXOFF_CONFIG_VLAN) ?
770 (olx & MLX5_TXOFF_CONFIG_METADATA) ?
772 (olx & MLX5_TXOFF_CONFIG_TXPP) ?
774 (txq && txq->fast_free) ?
775 " + Fast Free" : "");