From: Michael Baum <michaelba@nvidia.com>
Date: Mon, 12 Apr 2021 06:32:24 +0000 (+0000)
Subject: net/mlx5: separate Tx function implementations to new file
X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=79a876e3c4ef2cafd8f20d37f94732a9bbc54d17;p=dpdk.git

net/mlx5: separate Tx function implementations to new file

This patch separates Tx function implementations to different source
file as an optional preparation step for Tx cleanup.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---

diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index 69b285cdad..fac4728b4b 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -25,6 +25,7 @@ sources = files(
 	'mlx5_rxtx.c',
 	'mlx5_stats.c',
 	'mlx5_trigger.c',
+	'mlx5_tx.c',
 	'mlx5_txq.c',
 	'mlx5_txpp.c',
 	'mlx5_vlan.c',
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 2f36754b57..35c4cc330f 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -28,8 +28,6 @@
 #include "mlx5_rx.h"
 #include "mlx5_tx.h"
 
-#define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
-
 /* static asserts */
 static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
 static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
@@ -356,113 +354,6 @@ mlx5_dump_debug_information(const char *fname, const char *hex_title,
 	fclose(fd);
 }
 
-/**
- * Move QP from error state to running state and initialize indexes.
- *
- * @param txq_ctrl
- *   Pointer to TX queue control structure.
- *
- * @return
- *   0 on success, else -1.
- */
-static int
-tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl)
-{
-	struct mlx5_mp_arg_queue_state_modify sm = {
-			.is_wq = 0,
-			.queue_id = txq_ctrl->txq.idx,
-	};
-
-	if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm))
-		return -1;
-	txq_ctrl->txq.wqe_ci = 0;
-	txq_ctrl->txq.wqe_pi = 0;
-	txq_ctrl->txq.elts_comp = 0;
-	return 0;
-}
-
-/* Return 1 if the error CQE is signed otherwise, sign it and return 0. */
-static int
-check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe)
-{
-	static const uint8_t magic[] = "seen";
-	int ret = 1;
-	unsigned int i;
-
-	for (i = 0; i < sizeof(magic); ++i)
-		if (!ret || err_cqe->rsvd1[i] != magic[i]) {
-			ret = 0;
-			err_cqe->rsvd1[i] = magic[i];
-		}
-	return ret;
-}
-
-/**
- * Handle error CQE.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param error_cqe
- *   Pointer to the error CQE.
- *
- * @return
- *   Negative value if queue recovery failed, otherwise
- *   the error completion entry is handled successfully.
- */
-static int
-mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq,
-			 volatile struct mlx5_err_cqe *err_cqe)
-{
-	if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {
-		const uint16_t wqe_m = ((1 << txq->wqe_n) - 1);
-		struct mlx5_txq_ctrl *txq_ctrl =
-				container_of(txq, struct mlx5_txq_ctrl, txq);
-		uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter);
-		int seen = check_err_cqe_seen(err_cqe);
-
-		if (!seen && txq_ctrl->dump_file_n <
-		    txq_ctrl->priv->config.max_dump_files_num) {
-			MKSTR(err_str, "Unexpected CQE error syndrome "
-			      "0x%02x CQN = %u SQN = %u wqe_counter = %u "
-			      "wq_ci = %u cq_ci = %u", err_cqe->syndrome,
-			      txq->cqe_s, txq->qp_num_8s >> 8,
-			      rte_be_to_cpu_16(err_cqe->wqe_counter),
-			      txq->wqe_ci, txq->cq_ci);
-			MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u",
-			      PORT_ID(txq_ctrl->priv), txq->idx,
-			      txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc());
-			mlx5_dump_debug_information(name, NULL, err_str, 0);
-			mlx5_dump_debug_information(name, "MLX5 Error CQ:",
-						    (const void *)((uintptr_t)
-						    txq->cqes),
-						    sizeof(*err_cqe) *
-						    (1 << txq->cqe_n));
-			mlx5_dump_debug_information(name, "MLX5 Error SQ:",
-						    (const void *)((uintptr_t)
-						    txq->wqes),
-						    MLX5_WQE_SIZE *
-						    (1 << txq->wqe_n));
-			txq_ctrl->dump_file_n++;
-		}
-		if (!seen)
-			/*
-			 * Count errors in WQEs units.
-			 * Later it can be improved to count error packets,
-			 * for example, by SQ parsing to find how much packets
-			 * should be counted for each WQE.
-			 */
-			txq->stats.oerrors += ((txq->wqe_ci & wqe_m) -
-						new_wqe_pi) & wqe_m;
-		if (tx_recover_qp(txq_ctrl)) {
-			/* Recovering failed - retry later on the same WQE. */
-			return -1;
-		}
-		/* Release all the remaining buffers. */
-		txq_free_elts(txq_ctrl);
-	}
-	return 0;
-}
-
 /**
  * Modify a Verbs/DevX queue state.
  * This must be called from the primary process.
@@ -539,174 +430,6 @@ mlx5_queue_state_modify(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Dummy DPDK callback for TX.
- *
- * This function is used to temporarily replace the real callback during
- * unsafe control operations on the queue, or in case of error.
- *
- * @param dpdk_txq
- *   Generic pointer to TX queue structure.
- * @param[in] pkts
- *   Packets to transmit.
- * @param pkts_n
- *   Number of packets in array.
- *
- * @return
- *   Number of packets successfully transmitted (<= pkts_n).
- */
-uint16_t
-removed_tx_burst(void *dpdk_txq __rte_unused,
-		 struct rte_mbuf **pkts __rte_unused,
-		 uint16_t pkts_n __rte_unused)
-{
-	rte_mb();
-	return 0;
-}
-
-/**
- * Update completion queue consuming index via doorbell
- * and flush the completed data buffers.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param valid CQE pointer
- *   if not NULL update txq->wqe_pi and flush the buffers
- * @param olx
- *   Configured Tx offloads mask. It is fully defined at
- *   compile time and may be used for optimization.
- */
-static __rte_always_inline void
-mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq,
-		   volatile struct mlx5_cqe *last_cqe,
-		   unsigned int olx __rte_unused)
-{
-	if (likely(last_cqe != NULL)) {
-		uint16_t tail;
-
-		txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter);
-		tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m];
-		if (likely(tail != txq->elts_tail)) {
-			mlx5_tx_free_elts(txq, tail, olx);
-			MLX5_ASSERT(tail == txq->elts_tail);
-		}
-	}
-}
-
-/**
- * Manage TX completions. This routine checks the CQ for
- * arrived CQEs, deduces the last accomplished WQE in SQ,
- * updates SQ producing index and frees all completed mbufs.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param olx
- *   Configured Tx offloads mask. It is fully defined at
- *   compile time and may be used for optimization.
- *
- * NOTE: not inlined intentionally, it makes tx_burst
- * routine smaller, simple and faster - from experiments.
- */
-void
-mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq,
-			  unsigned int olx __rte_unused)
-{
-	unsigned int count = MLX5_TX_COMP_MAX_CQE;
-	volatile struct mlx5_cqe *last_cqe = NULL;
-	bool ring_doorbell = false;
-	int ret;
-
-	do {
-		volatile struct mlx5_cqe *cqe;
-
-		cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
-		ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
-		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
-			if (likely(ret != MLX5_CQE_STATUS_ERR)) {
-				/* No new CQEs in completion queue. */
-				MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN);
-				break;
-			}
-			/*
-			 * Some error occurred, try to restart.
-			 * We have no barrier after WQE related Doorbell
-			 * written, make sure all writes are completed
-			 * here, before we might perform SQ reset.
-			 */
-			rte_wmb();
-			ret = mlx5_tx_error_cqe_handle
-				(txq, (volatile struct mlx5_err_cqe *)cqe);
-			if (unlikely(ret < 0)) {
-				/*
-				 * Some error occurred on queue error
-				 * handling, we do not advance the index
-				 * here, allowing to retry on next call.
-				 */
-				return;
-			}
-			/*
-			 * We are going to fetch all entries with
-			 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status.
-			 * The send queue is supposed to be empty.
-			 */
-			ring_doorbell = true;
-			++txq->cq_ci;
-			txq->cq_pi = txq->cq_ci;
-			last_cqe = NULL;
-			continue;
-		}
-		/* Normal transmit completion. */
-		MLX5_ASSERT(txq->cq_ci != txq->cq_pi);
-#ifdef RTE_LIBRTE_MLX5_DEBUG
-		MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
-			    cqe->wqe_counter);
-#endif
-		ring_doorbell = true;
-		++txq->cq_ci;
-		last_cqe = cqe;
-		/*
-		 * We have to restrict the amount of processed CQEs
-		 * in one tx_burst routine call. The CQ may be large
-		 * and many CQEs may be updated by the NIC in one
-		 * transaction. Buffers freeing is time consuming,
-		 * multiple iterations may introduce significant
-		 * latency.
-		 */
-		if (likely(--count == 0))
-			break;
-	} while (true);
-	if (likely(ring_doorbell)) {
-		/* Ring doorbell to notify hardware. */
-		rte_compiler_barrier();
-		*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
-		mlx5_tx_comp_flush(txq, last_cqe, olx);
-	}
-}
-
-/**
- * DPDK callback to check the status of a tx descriptor.
- *
- * @param tx_queue
- *   The tx queue.
- * @param[in] offset
- *   The index of the descriptor in the ring.
- *
- * @return
- *   The status of the tx descriptor.
- */
-int
-mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
-{
-	struct mlx5_txq_data *__rte_restrict txq = tx_queue;
-	uint16_t used;
-
-	mlx5_tx_handle_completion(txq, 0);
-	used = txq->elts_head - txq->elts_tail;
-	if (offset < used)
-		return RTE_ETH_TX_DESC_FULL;
-	return RTE_ETH_TX_DESC_DONE;
-}
-
 /* Generate routines with Enhanced Multi-Packet Write support. */
 MLX5_TXOFF_DECL(full_empw,
 		MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_EMPW)
@@ -907,483 +630,3 @@ MLX5_TXOFF_DECL(mc_mpw,
 MLX5_TXOFF_DECL(i_mpw,
 		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
 		MLX5_TXOFF_CONFIG_MPW)
-
-/*
- * Array of declared and compiled Tx burst function and corresponding
- * supported offloads set. The array is used to select the Tx burst
- * function for specified offloads set at Tx queue configuration time.
- */
-const struct {
-	eth_tx_burst_t func;
-	unsigned int olx;
-} txoff_func[] = {
-MLX5_TXOFF_INFO(full_empw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(none_empw,
-		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(md_empw,
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mt_empw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mtsc_empw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mti_empw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_INLINE |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mtv_empw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mtiv_empw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(sc_empw,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(sci_empw,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(scv_empw,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(sciv_empw,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(i_empw,
-		MLX5_TXOFF_CONFIG_INLINE |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(v_empw,
-		MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(iv_empw,
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(full_ts_nompw,
-		MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP)
-
-MLX5_TXOFF_INFO(full_ts_nompwi,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
-		MLX5_TXOFF_CONFIG_TXPP)
-
-MLX5_TXOFF_INFO(full_ts,
-		MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP |
-		MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(full_ts_noi,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
-		MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(none_ts,
-		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP |
-		MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mdi_ts,
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
-		MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mti_ts,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
-		MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(mtiv_ts,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP |
-		MLX5_TXOFF_CONFIG_EMPW)
-
-MLX5_TXOFF_INFO(full,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(none,
-		MLX5_TXOFF_CONFIG_NONE)
-
-MLX5_TXOFF_INFO(md,
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(mt,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(mtsc,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(mti,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_INLINE |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(mtv,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(mtiv,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(sc,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(sci,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(scv,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(sciv,
-		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(i,
-		MLX5_TXOFF_CONFIG_INLINE |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(v,
-		MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(iv,
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
-		MLX5_TXOFF_CONFIG_METADATA)
-
-MLX5_TXOFF_INFO(none_mpw,
-		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW |
-		MLX5_TXOFF_CONFIG_MPW)
-
-MLX5_TXOFF_INFO(mci_mpw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
-		MLX5_TXOFF_CONFIG_MPW)
-
-MLX5_TXOFF_INFO(mc_mpw,
-		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
-		MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW)
-
-MLX5_TXOFF_INFO(i_mpw,
-		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
-		MLX5_TXOFF_CONFIG_MPW)
-};
-
-/**
- * Configure the Tx function to use. The routine checks configured
- * Tx offloads for the device and selects appropriate Tx burst
- * routine. There are multiple Tx burst routines compiled from
- * the same template in the most optimal way for the dedicated
- * Tx offloads set.
- *
- * @param dev
- *   Pointer to private data structure.
- *
- * @return
- *   Pointer to selected Tx burst function.
- */
-eth_tx_burst_t
-mlx5_select_tx_function(struct rte_eth_dev *dev)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
-	uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
-	unsigned int diff = 0, olx = 0, i, m;
-
-	MLX5_ASSERT(priv);
-	if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
-		/* We should support Multi-Segment Packets. */
-		olx |= MLX5_TXOFF_CONFIG_MULTI;
-	}
-	if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
-			   DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
-			   DEV_TX_OFFLOAD_GRE_TNL_TSO |
-			   DEV_TX_OFFLOAD_IP_TNL_TSO |
-			   DEV_TX_OFFLOAD_UDP_TNL_TSO)) {
-		/* We should support TCP Send Offload. */
-		olx |= MLX5_TXOFF_CONFIG_TSO;
-	}
-	if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
-			   DEV_TX_OFFLOAD_UDP_TNL_TSO |
-			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
-		/* We should support Software Parser for Tunnels. */
-		olx |= MLX5_TXOFF_CONFIG_SWP;
-	}
-	if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
-			   DEV_TX_OFFLOAD_UDP_CKSUM |
-			   DEV_TX_OFFLOAD_TCP_CKSUM |
-			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
-		/* We should support IP/TCP/UDP Checksums. */
-		olx |= MLX5_TXOFF_CONFIG_CSUM;
-	}
-	if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) {
-		/* We should support VLAN insertion. */
-		olx |= MLX5_TXOFF_CONFIG_VLAN;
-	}
-	if (tx_offloads & DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP &&
-	    rte_mbuf_dynflag_lookup
-			(RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 &&
-	    rte_mbuf_dynfield_lookup
-			(RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) {
-		/* Offload configured, dynamic entities registered. */
-		olx |= MLX5_TXOFF_CONFIG_TXPP;
-	}
-	if (priv->txqs_n && (*priv->txqs)[0]) {
-		struct mlx5_txq_data *txd = (*priv->txqs)[0];
-
-		if (txd->inlen_send) {
-			/*
-			 * Check the data inline requirements. Data inline
-			 * is enabled on per device basis, we can check
-			 * the first Tx queue only.
-			 *
-			 * If device does not support VLAN insertion in WQE
-			 * and some queues are requested to perform VLAN
-			 * insertion offload than inline must be enabled.
-			 */
-			olx |= MLX5_TXOFF_CONFIG_INLINE;
-		}
-	}
-	if (config->mps == MLX5_MPW_ENHANCED &&
-	    config->txq_inline_min <= 0) {
-		/*
-		 * The NIC supports Enhanced Multi-Packet Write
-		 * and does not require minimal inline data.
-		 */
-		olx |= MLX5_TXOFF_CONFIG_EMPW;
-	}
-	if (rte_flow_dynf_metadata_avail()) {
-		/* We should support Flow metadata. */
-		olx |= MLX5_TXOFF_CONFIG_METADATA;
-	}
-	if (config->mps == MLX5_MPW) {
-		/*
-		 * The NIC supports Legacy Multi-Packet Write.
-		 * The MLX5_TXOFF_CONFIG_MPW controls the
-		 * descriptor building method in combination
-		 * with MLX5_TXOFF_CONFIG_EMPW.
-		 */
-		if (!(olx & (MLX5_TXOFF_CONFIG_TSO |
-			     MLX5_TXOFF_CONFIG_SWP |
-			     MLX5_TXOFF_CONFIG_VLAN |
-			     MLX5_TXOFF_CONFIG_METADATA)))
-			olx |= MLX5_TXOFF_CONFIG_EMPW |
-			       MLX5_TXOFF_CONFIG_MPW;
-	}
-	/*
-	 * Scan the routines table to find the minimal
-	 * satisfying routine with requested offloads.
-	 */
-	m = RTE_DIM(txoff_func);
-	for (i = 0; i < RTE_DIM(txoff_func); i++) {
-		unsigned int tmp;
-
-		tmp = txoff_func[i].olx;
-		if (tmp == olx) {
-			/* Meets requested offloads exactly.*/
-			m = i;
-			break;
-		}
-		if ((tmp & olx) != olx) {
-			/* Does not meet requested offloads at all. */
-			continue;
-		}
-		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW)
-			/* Do not enable legacy MPW if not configured. */
-			continue;
-		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
-			/* Do not enable eMPW if not configured. */
-			continue;
-		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
-			/* Do not enable inlining if not configured. */
-			continue;
-		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP)
-			/* Do not enable scheduling if not configured. */
-			continue;
-		/*
-		 * Some routine meets the requirements.
-		 * Check whether it has minimal amount
-		 * of not requested offloads.
-		 */
-		tmp = __builtin_popcountl(tmp & ~olx);
-		if (m >= RTE_DIM(txoff_func) || tmp < diff) {
-			/* First or better match, save and continue. */
-			m = i;
-			diff = tmp;
-			continue;
-		}
-		if (tmp == diff) {
-			tmp = txoff_func[i].olx ^ txoff_func[m].olx;
-			if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
-			    __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
-				/* Lighter not requested offload. */
-				m = i;
-			}
-		}
-	}
-	if (m >= RTE_DIM(txoff_func)) {
-		DRV_LOG(DEBUG, "port %u has no selected Tx function"
-			       " for requested offloads %04X",
-				dev->data->port_id, olx);
-		return NULL;
-	}
-	DRV_LOG(DEBUG, "port %u has selected Tx function"
-		       " supporting offloads %04X/%04X",
-			dev->data->port_id, olx, txoff_func[m].olx);
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
-		DRV_LOG(DEBUG, "\tMULTI (multi segment)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
-		DRV_LOG(DEBUG, "\tTSO   (TCP send offload)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
-		DRV_LOG(DEBUG, "\tSWP   (software parser)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
-		DRV_LOG(DEBUG, "\tCSUM  (checksum offload)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
-		DRV_LOG(DEBUG, "\tINLIN (inline data)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
-		DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
-		DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP)
-		DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)");
-	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) {
-		if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW)
-			DRV_LOG(DEBUG, "\tMPW   (Legacy MPW)");
-		else
-			DRV_LOG(DEBUG, "\tEMPW  (Enhanced MPW)");
-	}
-	return txoff_func[m].func;
-}
-
-/**
- * DPDK callback to get the TX queue information
- *
- * @param dev
- *   Pointer to the device structure.
- *
- * @param tx_queue_id
- *   Tx queue identificator.
- *
- * @param qinfo
- *   Pointer to the TX queue information structure.
- *
- * @return
- *   None.
- */
-
-void
-mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
-		  struct rte_eth_txq_info *qinfo)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
-	struct mlx5_txq_ctrl *txq_ctrl =
-			container_of(txq, struct mlx5_txq_ctrl, txq);
-
-	if (!txq)
-		return;
-	qinfo->nb_desc = txq->elts_s;
-	qinfo->conf.tx_thresh.pthresh = 0;
-	qinfo->conf.tx_thresh.hthresh = 0;
-	qinfo->conf.tx_thresh.wthresh = 0;
-	qinfo->conf.tx_rs_thresh = 0;
-	qinfo->conf.tx_free_thresh = 0;
-	qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1;
-	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
-}
-
-/**
- * DPDK callback to get the TX packet burst mode information
- *
- * @param dev
- *   Pointer to the device structure.
- *
- * @param tx_queue_id
- *   Tx queue identificatior.
- *
- * @param mode
- *   Pointer to the burts mode information.
- *
- * @return
- *   0 as success, -EINVAL as failure.
- */
-
-int
-mlx5_tx_burst_mode_get(struct rte_eth_dev *dev,
-		       uint16_t tx_queue_id,
-		       struct rte_eth_burst_mode *mode)
-{
-	eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
-	unsigned int i, olx;
-
-	for (i = 0; i < RTE_DIM(txoff_func); i++) {
-		if (pkt_burst == txoff_func[i].func) {
-			olx = txoff_func[i].olx;
-			snprintf(mode->info, sizeof(mode->info),
-				 "%s%s%s%s%s%s%s%s%s%s",
-				 (olx & MLX5_TXOFF_CONFIG_EMPW) ?
-				 ((olx & MLX5_TXOFF_CONFIG_MPW) ?
-				 "Legacy MPW" : "Enhanced MPW") : "No MPW",
-				 (olx & MLX5_TXOFF_CONFIG_MULTI) ?
-				 " + MULTI" : "",
-				 (olx & MLX5_TXOFF_CONFIG_TSO) ?
-				 " + TSO" : "",
-				 (olx & MLX5_TXOFF_CONFIG_SWP) ?
-				 " + SWP" : "",
-				 (olx & MLX5_TXOFF_CONFIG_CSUM) ?
-				 "  + CSUM" : "",
-				 (olx & MLX5_TXOFF_CONFIG_INLINE) ?
-				 " + INLINE" : "",
-				 (olx & MLX5_TXOFF_CONFIG_VLAN) ?
-				 " + VLAN" : "",
-				 (olx & MLX5_TXOFF_CONFIG_METADATA) ?
-				 " + METADATA" : "",
-				 (olx & MLX5_TXOFF_CONFIG_TXPP) ?
-				 " + TXPP" : "",
-				 (txq && txq->fast_free) ?
-				 " + Fast Free" : "");
-			return 0;
-		}
-	}
-	return -EINVAL;
-}
diff --git a/drivers/net/mlx5/mlx5_tx.c b/drivers/net/mlx5/mlx5_tx.c
new file mode 100644
index 0000000000..df671379e4
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_tx.c
@@ -0,0 +1,780 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 6WIND S.A.
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_cycles.h>
+#include <rte_flow.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_common.h>
+
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5.h"
+#include "mlx5_mr.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_tx.h"
+
+#define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
+
+/**
+ * Move QP from error state to running state and initialize indexes.
+ *
+ * @param txq_ctrl
+ *   Pointer to TX queue control structure.
+ *
+ * @return
+ *   0 on success, else -1.
+ */
+static int
+tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl)
+{
+	struct mlx5_mp_arg_queue_state_modify sm = {
+			.is_wq = 0,
+			.queue_id = txq_ctrl->txq.idx,
+	};
+
+	if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm))
+		return -1;
+	txq_ctrl->txq.wqe_ci = 0;
+	txq_ctrl->txq.wqe_pi = 0;
+	txq_ctrl->txq.elts_comp = 0;
+	return 0;
+}
+
+/* Return 1 if the error CQE is signed otherwise, sign it and return 0. */
+static int
+check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe)
+{
+	static const uint8_t magic[] = "seen";
+	int ret = 1;
+	unsigned int i;
+
+	for (i = 0; i < sizeof(magic); ++i)
+		if (!ret || err_cqe->rsvd1[i] != magic[i]) {
+			ret = 0;
+			err_cqe->rsvd1[i] = magic[i];
+		}
+	return ret;
+}
+
+/**
+ * Handle error CQE.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param error_cqe
+ *   Pointer to the error CQE.
+ *
+ * @return
+ *   Negative value if queue recovery failed, otherwise
+ *   the error completion entry is handled successfully.
+ */
+static int
+mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq,
+			 volatile struct mlx5_err_cqe *err_cqe)
+{
+	if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {
+		const uint16_t wqe_m = ((1 << txq->wqe_n) - 1);
+		struct mlx5_txq_ctrl *txq_ctrl =
+				container_of(txq, struct mlx5_txq_ctrl, txq);
+		uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter);
+		int seen = check_err_cqe_seen(err_cqe);
+
+		if (!seen && txq_ctrl->dump_file_n <
+		    txq_ctrl->priv->config.max_dump_files_num) {
+			MKSTR(err_str, "Unexpected CQE error syndrome "
+			      "0x%02x CQN = %u SQN = %u wqe_counter = %u "
+			      "wq_ci = %u cq_ci = %u", err_cqe->syndrome,
+			      txq->cqe_s, txq->qp_num_8s >> 8,
+			      rte_be_to_cpu_16(err_cqe->wqe_counter),
+			      txq->wqe_ci, txq->cq_ci);
+			MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u",
+			      PORT_ID(txq_ctrl->priv), txq->idx,
+			      txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc());
+			mlx5_dump_debug_information(name, NULL, err_str, 0);
+			mlx5_dump_debug_information(name, "MLX5 Error CQ:",
+						    (const void *)((uintptr_t)
+						    txq->cqes),
+						    sizeof(*err_cqe) *
+						    (1 << txq->cqe_n));
+			mlx5_dump_debug_information(name, "MLX5 Error SQ:",
+						    (const void *)((uintptr_t)
+						    txq->wqes),
+						    MLX5_WQE_SIZE *
+						    (1 << txq->wqe_n));
+			txq_ctrl->dump_file_n++;
+		}
+		if (!seen)
+			/*
+			 * Count errors in WQEs units.
+			 * Later it can be improved to count error packets,
+			 * for example, by SQ parsing to find how much packets
+			 * should be counted for each WQE.
+			 */
+			txq->stats.oerrors += ((txq->wqe_ci & wqe_m) -
+						new_wqe_pi) & wqe_m;
+		if (tx_recover_qp(txq_ctrl)) {
+			/* Recovering failed - retry later on the same WQE. */
+			return -1;
+		}
+		/* Release all the remaining buffers. */
+		txq_free_elts(txq_ctrl);
+	}
+	return 0;
+}
+
+/**
+ * Dummy DPDK callback for TX.
+ *
+ * This function is used to temporarily replace the real callback during
+ * unsafe control operations on the queue, or in case of error.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+removed_tx_burst(void *dpdk_txq __rte_unused,
+		 struct rte_mbuf **pkts __rte_unused,
+		 uint16_t pkts_n __rte_unused)
+{
+	rte_mb();
+	return 0;
+}
+
+/**
+ * Update completion queue consuming index via doorbell
+ * and flush the completed data buffers.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param last_cqe
+ *   valid CQE pointer, if not NULL update txq->wqe_pi and flush the buffers.
+ * @param olx
+ *   Configured Tx offloads mask. It is fully defined at
+ *   compile time and may be used for optimization.
+ */
+static __rte_always_inline void
+mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq,
+		   volatile struct mlx5_cqe *last_cqe,
+		   unsigned int olx __rte_unused)
+{
+	if (likely(last_cqe != NULL)) {
+		uint16_t tail;
+
+		txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter);
+		tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m];
+		if (likely(tail != txq->elts_tail)) {
+			mlx5_tx_free_elts(txq, tail, olx);
+			MLX5_ASSERT(tail == txq->elts_tail);
+		}
+	}
+}
+
+/**
+ * Manage TX completions. This routine checks the CQ for
+ * arrived CQEs, deduces the last accomplished WQE in SQ,
+ * updates SQ producing index and frees all completed mbufs.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param olx
+ *   Configured Tx offloads mask. It is fully defined at
+ *   compile time and may be used for optimization.
+ *
+ * NOTE: not inlined intentionally, it makes tx_burst
+ * routine smaller, simple and faster - from experiments.
+ */
+void
+mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq,
+			  unsigned int olx __rte_unused)
+{
+	unsigned int count = MLX5_TX_COMP_MAX_CQE;
+	volatile struct mlx5_cqe *last_cqe = NULL;
+	bool ring_doorbell = false;
+	int ret;
+
+	do {
+		volatile struct mlx5_cqe *cqe;
+
+		cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
+		ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (likely(ret != MLX5_CQE_STATUS_ERR)) {
+				/* No new CQEs in completion queue. */
+				MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN);
+				break;
+			}
+			/*
+			 * Some error occurred, try to restart.
+			 * We have no barrier after WQE related Doorbell
+			 * written, make sure all writes are completed
+			 * here, before we might perform SQ reset.
+			 */
+			rte_wmb();
+			ret = mlx5_tx_error_cqe_handle
+				(txq, (volatile struct mlx5_err_cqe *)cqe);
+			if (unlikely(ret < 0)) {
+				/*
+				 * Some error occurred on queue error
+				 * handling, we do not advance the index
+				 * here, allowing to retry on next call.
+				 */
+				return;
+			}
+			/*
+			 * We are going to fetch all entries with
+			 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status.
+			 * The send queue is supposed to be empty.
+			 */
+			ring_doorbell = true;
+			++txq->cq_ci;
+			txq->cq_pi = txq->cq_ci;
+			last_cqe = NULL;
+			continue;
+		}
+		/* Normal transmit completion. */
+		MLX5_ASSERT(txq->cq_ci != txq->cq_pi);
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
+			    cqe->wqe_counter);
+#endif
+		ring_doorbell = true;
+		++txq->cq_ci;
+		last_cqe = cqe;
+		/*
+		 * We have to restrict the amount of processed CQEs
+		 * in one tx_burst routine call. The CQ may be large
+		 * and many CQEs may be updated by the NIC in one
+		 * transaction. Buffers freeing is time consuming,
+		 * multiple iterations may introduce significant latency.
+		 */
+		if (likely(--count == 0))
+			break;
+	} while (true);
+	if (likely(ring_doorbell)) {
+		/* Ring doorbell to notify hardware. */
+		rte_compiler_barrier();
+		*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
+		mlx5_tx_comp_flush(txq, last_cqe, olx);
+	}
+}
+
+/**
+ * DPDK callback to check the status of a Tx descriptor.
+ *
+ * @param tx_queue
+ *   The Tx queue.
+ * @param[in] offset
+ *   The index of the descriptor in the ring.
+ *
+ * @return
+ *   The status of the Tx descriptor.
+ */
+int
+mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
+{
+	struct mlx5_txq_data *__rte_restrict txq = tx_queue;
+	uint16_t used;
+
+	mlx5_tx_handle_completion(txq, 0);
+	used = txq->elts_head - txq->elts_tail;
+	if (offset < used)
+		return RTE_ETH_TX_DESC_FULL;
+	return RTE_ETH_TX_DESC_DONE;
+}
+
+/*
+ * Array of declared and compiled Tx burst function and corresponding
+ * supported offloads set. The array is used to select the Tx burst
+ * function for specified offloads set at Tx queue configuration time.
+ */
+const struct {
+	eth_tx_burst_t func;
+	unsigned int olx;
+} txoff_func[] = {
+MLX5_TXOFF_INFO(full_empw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(none_empw,
+		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(md_empw,
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mt_empw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mtsc_empw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mti_empw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_INLINE |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mtv_empw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mtiv_empw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(sc_empw,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(sci_empw,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(scv_empw,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(sciv_empw,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(i_empw,
+		MLX5_TXOFF_CONFIG_INLINE |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(v_empw,
+		MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(iv_empw,
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(full_ts_nompw,
+		MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP)
+
+MLX5_TXOFF_INFO(full_ts_nompwi,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
+		MLX5_TXOFF_CONFIG_TXPP)
+
+MLX5_TXOFF_INFO(full_ts,
+		MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP |
+		MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(full_ts_noi,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA |
+		MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(none_ts,
+		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP |
+		MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mdi_ts,
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
+		MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mti_ts,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA |
+		MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(mtiv_ts,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP |
+		MLX5_TXOFF_CONFIG_EMPW)
+
+MLX5_TXOFF_INFO(full,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(none,
+		MLX5_TXOFF_CONFIG_NONE)
+
+MLX5_TXOFF_INFO(md,
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(mt,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(mtsc,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(mti,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_INLINE |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(mtv,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(mtiv,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(sc,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(sci,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(scv,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(sciv,
+		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(i,
+		MLX5_TXOFF_CONFIG_INLINE |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(v,
+		MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(iv,
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
+		MLX5_TXOFF_CONFIG_METADATA)
+
+MLX5_TXOFF_INFO(none_mpw,
+		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW |
+		MLX5_TXOFF_CONFIG_MPW)
+
+MLX5_TXOFF_INFO(mci_mpw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
+		MLX5_TXOFF_CONFIG_MPW)
+
+MLX5_TXOFF_INFO(mc_mpw,
+		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM |
+		MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW)
+
+MLX5_TXOFF_INFO(i_mpw,
+		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW |
+		MLX5_TXOFF_CONFIG_MPW)
+};
+
+/**
+ * Configure the Tx function to use. The routine checks configured
+ * Tx offloads for the device and selects appropriate Tx burst routine.
+ * There are multiple Tx burst routines compiled from the same template
+ * in the most optimal way for the dedicated Tx offloads set.
+ *
+ * @param dev
+ *   Pointer to private data structure.
+ *
+ * @return
+ *   Pointer to selected Tx burst function.
+ */
+eth_tx_burst_t
+mlx5_select_tx_function(struct rte_eth_dev *dev)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *config = &priv->config;
+	uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
+	unsigned int diff = 0, olx = 0, i, m;
+
+	MLX5_ASSERT(priv);
+	if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
+		/* We should support Multi-Segment Packets. */
+		olx |= MLX5_TXOFF_CONFIG_MULTI;
+	}
+	if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
+			   DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
+			   DEV_TX_OFFLOAD_GRE_TNL_TSO |
+			   DEV_TX_OFFLOAD_IP_TNL_TSO |
+			   DEV_TX_OFFLOAD_UDP_TNL_TSO)) {
+		/* We should support TCP Send Offload. */
+		olx |= MLX5_TXOFF_CONFIG_TSO;
+	}
+	if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
+			   DEV_TX_OFFLOAD_UDP_TNL_TSO |
+			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
+		/* We should support Software Parser for Tunnels. */
+		olx |= MLX5_TXOFF_CONFIG_SWP;
+	}
+	if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
+			   DEV_TX_OFFLOAD_UDP_CKSUM |
+			   DEV_TX_OFFLOAD_TCP_CKSUM |
+			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
+		/* We should support IP/TCP/UDP Checksums. */
+		olx |= MLX5_TXOFF_CONFIG_CSUM;
+	}
+	if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) {
+		/* We should support VLAN insertion. */
+		olx |= MLX5_TXOFF_CONFIG_VLAN;
+	}
+	if (tx_offloads & DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP &&
+	    rte_mbuf_dynflag_lookup
+			(RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 &&
+	    rte_mbuf_dynfield_lookup
+			(RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) {
+		/* Offload configured, dynamic entities registered. */
+		olx |= MLX5_TXOFF_CONFIG_TXPP;
+	}
+	if (priv->txqs_n && (*priv->txqs)[0]) {
+		struct mlx5_txq_data *txd = (*priv->txqs)[0];
+
+		if (txd->inlen_send) {
+			/*
+			 * Check the data inline requirements. Data inline
+			 * is enabled on per device basis, we can check
+			 * the first Tx queue only.
+			 *
+			 * If device does not support VLAN insertion in WQE
+			 * and some queues are requested to perform VLAN
+			 * insertion offload than inline must be enabled.
+			 */
+			olx |= MLX5_TXOFF_CONFIG_INLINE;
+		}
+	}
+	if (config->mps == MLX5_MPW_ENHANCED &&
+	    config->txq_inline_min <= 0) {
+		/*
+		 * The NIC supports Enhanced Multi-Packet Write
+		 * and does not require minimal inline data.
+		 */
+		olx |= MLX5_TXOFF_CONFIG_EMPW;
+	}
+	if (rte_flow_dynf_metadata_avail()) {
+		/* We should support Flow metadata. */
+		olx |= MLX5_TXOFF_CONFIG_METADATA;
+	}
+	if (config->mps == MLX5_MPW) {
+		/*
+		 * The NIC supports Legacy Multi-Packet Write.
+		 * The MLX5_TXOFF_CONFIG_MPW controls the descriptor building
+		 * method in combination with MLX5_TXOFF_CONFIG_EMPW.
+		 */
+		if (!(olx & (MLX5_TXOFF_CONFIG_TSO |
+			     MLX5_TXOFF_CONFIG_SWP |
+			     MLX5_TXOFF_CONFIG_VLAN |
+			     MLX5_TXOFF_CONFIG_METADATA)))
+			olx |= MLX5_TXOFF_CONFIG_EMPW |
+			       MLX5_TXOFF_CONFIG_MPW;
+	}
+	/*
+	 * Scan the routines table to find the minimal
+	 * satisfying routine with requested offloads.
+	 */
+	m = RTE_DIM(txoff_func);
+	for (i = 0; i < RTE_DIM(txoff_func); i++) {
+		unsigned int tmp;
+
+		tmp = txoff_func[i].olx;
+		if (tmp == olx) {
+			/* Meets requested offloads exactly.*/
+			m = i;
+			break;
+		}
+		if ((tmp & olx) != olx) {
+			/* Does not meet requested offloads at all. */
+			continue;
+		}
+		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW)
+			/* Do not enable legacy MPW if not configured. */
+			continue;
+		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
+			/* Do not enable eMPW if not configured. */
+			continue;
+		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
+			/* Do not enable inlining if not configured. */
+			continue;
+		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP)
+			/* Do not enable scheduling if not configured. */
+			continue;
+		/*
+		 * Some routine meets the requirements.
+		 * Check whether it has minimal amount
+		 * of not requested offloads.
+		 */
+		tmp = __builtin_popcountl(tmp & ~olx);
+		if (m >= RTE_DIM(txoff_func) || tmp < diff) {
+			/* First or better match, save and continue. */
+			m = i;
+			diff = tmp;
+			continue;
+		}
+		if (tmp == diff) {
+			tmp = txoff_func[i].olx ^ txoff_func[m].olx;
+			if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
+			    __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
+				/* Lighter not requested offload. */
+				m = i;
+			}
+		}
+	}
+	if (m >= RTE_DIM(txoff_func)) {
+		DRV_LOG(DEBUG, "port %u has no selected Tx function"
+			       " for requested offloads %04X",
+				dev->data->port_id, olx);
+		return NULL;
+	}
+	DRV_LOG(DEBUG, "port %u has selected Tx function"
+		       " supporting offloads %04X/%04X",
+			dev->data->port_id, olx, txoff_func[m].olx);
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
+		DRV_LOG(DEBUG, "\tMULTI (multi segment)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
+		DRV_LOG(DEBUG, "\tTSO   (TCP send offload)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
+		DRV_LOG(DEBUG, "\tSWP   (software parser)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
+		DRV_LOG(DEBUG, "\tCSUM  (checksum offload)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
+		DRV_LOG(DEBUG, "\tINLIN (inline data)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
+		DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
+		DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP)
+		DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)");
+	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) {
+		if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW)
+			DRV_LOG(DEBUG, "\tMPW   (Legacy MPW)");
+		else
+			DRV_LOG(DEBUG, "\tEMPW  (Enhanced MPW)");
+	}
+	return txoff_func[m].func;
+}
+
+/**
+ * DPDK callback to get the TX queue information.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ *
+ * @param tx_queue_id
+ *   Tx queue identificator.
+ *
+ * @param qinfo
+ *   Pointer to the TX queue information structure.
+ *
+ * @return
+ *   None.
+ */
+void
+mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+		  struct rte_eth_txq_info *qinfo)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
+	struct mlx5_txq_ctrl *txq_ctrl =
+			container_of(txq, struct mlx5_txq_ctrl, txq);
+
+	if (!txq)
+		return;
+	qinfo->nb_desc = txq->elts_s;
+	qinfo->conf.tx_thresh.pthresh = 0;
+	qinfo->conf.tx_thresh.hthresh = 0;
+	qinfo->conf.tx_thresh.wthresh = 0;
+	qinfo->conf.tx_rs_thresh = 0;
+	qinfo->conf.tx_free_thresh = 0;
+	qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1;
+	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
+}
+
+/**
+ * DPDK callback to get the TX packet burst mode information.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ *
+ * @param tx_queue_id
+ *   Tx queue identificatior.
+ *
+ * @param mode
+ *   Pointer to the burts mode information.
+ *
+ * @return
+ *   0 as success, -EINVAL as failure.
+ */
+int
+mlx5_tx_burst_mode_get(struct rte_eth_dev *dev,
+		       uint16_t tx_queue_id,
+		       struct rte_eth_burst_mode *mode)
+{
+	eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
+	unsigned int i, olx;
+
+	for (i = 0; i < RTE_DIM(txoff_func); i++) {
+		if (pkt_burst == txoff_func[i].func) {
+			olx = txoff_func[i].olx;
+			snprintf(mode->info, sizeof(mode->info),
+				 "%s%s%s%s%s%s%s%s%s%s",
+				 (olx & MLX5_TXOFF_CONFIG_EMPW) ?
+				 ((olx & MLX5_TXOFF_CONFIG_MPW) ?
+				 "Legacy MPW" : "Enhanced MPW") : "No MPW",
+				 (olx & MLX5_TXOFF_CONFIG_MULTI) ?
+				 " + MULTI" : "",
+				 (olx & MLX5_TXOFF_CONFIG_TSO) ?
+				 " + TSO" : "",
+				 (olx & MLX5_TXOFF_CONFIG_SWP) ?
+				 " + SWP" : "",
+				 (olx & MLX5_TXOFF_CONFIG_CSUM) ?
+				 "  + CSUM" : "",
+				 (olx & MLX5_TXOFF_CONFIG_INLINE) ?
+				 " + INLINE" : "",
+				 (olx & MLX5_TXOFF_CONFIG_VLAN) ?
+				 " + VLAN" : "",
+				 (olx & MLX5_TXOFF_CONFIG_METADATA) ?
+				 " + METADATA" : "",
+				 (olx & MLX5_TXOFF_CONFIG_TXPP) ?
+				 " + TXPP" : "",
+				 (txq && txq->fast_free) ?
+				 " + Fast Free" : "");
+			return 0;
+		}
+	}
+	return -EINVAL;
+}
diff --git a/drivers/net/mlx5/mlx5_tx.h b/drivers/net/mlx5/mlx5_tx.h
index 34843d473f..7a03aaf317 100644
--- a/drivers/net/mlx5/mlx5_tx.h
+++ b/drivers/net/mlx5/mlx5_tx.h
@@ -62,10 +62,15 @@ enum mlx5_txcmp_code {
 
 #define MLX5_TXOFF_CONFIG(mask) (olx & MLX5_TXOFF_CONFIG_##mask)
 
+#define MLX5_TXOFF_PRE_DECL(func) \
+uint16_t mlx5_tx_burst_##func(void *txq, \
+			      struct rte_mbuf **pkts, \
+			      uint16_t pkts_n)
+
 #define MLX5_TXOFF_DECL(func, olx) \
-static uint16_t mlx5_tx_burst_##func(void *txq, \
-				     struct rte_mbuf **pkts, \
-				    uint16_t pkts_n) \
+uint16_t mlx5_tx_burst_##func(void *txq, \
+			      struct rte_mbuf **pkts, \
+			      uint16_t pkts_n) \
 { \
 	return mlx5_tx_burst_tmpl((struct mlx5_txq_data *)txq, \
 		    pkts, pkts_n, (olx)); \
@@ -237,6 +242,54 @@ uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
 uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
 			       struct rte_mempool *mp);
 
+/* mlx5_rxtx.c */
+
+MLX5_TXOFF_PRE_DECL(full_empw);
+MLX5_TXOFF_PRE_DECL(none_empw);
+MLX5_TXOFF_PRE_DECL(md_empw);
+MLX5_TXOFF_PRE_DECL(mt_empw);
+MLX5_TXOFF_PRE_DECL(mtsc_empw);
+MLX5_TXOFF_PRE_DECL(mti_empw);
+MLX5_TXOFF_PRE_DECL(mtv_empw);
+MLX5_TXOFF_PRE_DECL(mtiv_empw);
+MLX5_TXOFF_PRE_DECL(sc_empw);
+MLX5_TXOFF_PRE_DECL(sci_empw);
+MLX5_TXOFF_PRE_DECL(scv_empw);
+MLX5_TXOFF_PRE_DECL(sciv_empw);
+MLX5_TXOFF_PRE_DECL(i_empw);
+MLX5_TXOFF_PRE_DECL(v_empw);
+MLX5_TXOFF_PRE_DECL(iv_empw);
+
+MLX5_TXOFF_PRE_DECL(full);
+MLX5_TXOFF_PRE_DECL(none);
+MLX5_TXOFF_PRE_DECL(md);
+MLX5_TXOFF_PRE_DECL(mt);
+MLX5_TXOFF_PRE_DECL(mtsc);
+MLX5_TXOFF_PRE_DECL(mti);
+MLX5_TXOFF_PRE_DECL(mtv);
+MLX5_TXOFF_PRE_DECL(mtiv);
+MLX5_TXOFF_PRE_DECL(sc);
+MLX5_TXOFF_PRE_DECL(sci);
+MLX5_TXOFF_PRE_DECL(scv);
+MLX5_TXOFF_PRE_DECL(sciv);
+MLX5_TXOFF_PRE_DECL(i);
+MLX5_TXOFF_PRE_DECL(v);
+MLX5_TXOFF_PRE_DECL(iv);
+
+MLX5_TXOFF_PRE_DECL(full_ts_nompw);
+MLX5_TXOFF_PRE_DECL(full_ts_nompwi);
+MLX5_TXOFF_PRE_DECL(full_ts);
+MLX5_TXOFF_PRE_DECL(full_ts_noi);
+MLX5_TXOFF_PRE_DECL(none_ts);
+MLX5_TXOFF_PRE_DECL(mdi_ts);
+MLX5_TXOFF_PRE_DECL(mti_ts);
+MLX5_TXOFF_PRE_DECL(mtiv_ts);
+
+MLX5_TXOFF_PRE_DECL(none_mpw);
+MLX5_TXOFF_PRE_DECL(mci_mpw);
+MLX5_TXOFF_PRE_DECL(mc_mpw);
+MLX5_TXOFF_PRE_DECL(i_mpw);
+
 static __rte_always_inline uint64_t *
 mlx5_tx_bfreg(struct mlx5_txq_data *txq)
 {
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 2bda2c8058..3e5e94444b 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -13,6 +13,7 @@
 #include <rte_mbuf.h>
 #include <rte_malloc.h>
 #include <ethdev_driver.h>
+#include <rte_bus_pci.h>
 #include <rte_common.h>
 #include <rte_eal_paging.h>