From 318ea4cfa1b1c1393181a0a70ba8fd9776de341a Mon Sep 17 00:00:00 2001
From: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Date: Mon, 29 Jul 2019 12:41:03 +0000
Subject: [PATCH] net/mlx5: fix Tx completion descriptors fetching loop

This patch limits the amount of fetched and processed
completion descriptors in one tx_burst routine call.

The completion processing involves the buffer freeing
which may be time consuming and introduce the significant
latency, so limiting the amount of processed completions
mitigates the latency issue.

Fixes: 18a1c20044c0 ("net/mlx5: implement Tx burst template")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5_defs.h |  7 ++++++
 drivers/net/mlx5/mlx5_rxtx.c | 46 ++++++++++++++++++++++++------------
 2 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 8c118d5bd0..461e916560 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -37,6 +37,13 @@
  */
 #define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
 
+/*
+ * Maximal amount of normal completion CQEs
+ * processed in one call of tx_burst() routine.
+ */
+#define MLX5_TX_COMP_MAX_CQE 2u
+
+
 /* Size of per-queue MR cache array for linear search. */
 #define MLX5_MR_CACHE_N 8
 
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 6627b54c9b..c7be532b94 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -2027,13 +2027,13 @@ static void
 mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq,
 			  unsigned int olx __rte_unused)
 {
+	unsigned int count = MLX5_TX_COMP_MAX_CQE;
 	bool update = false;
+	uint16_t tail = txq->elts_tail;
 	int ret;
 
 	do {
-		volatile struct mlx5_wqe_cseg *cseg;
 		volatile struct mlx5_cqe *cqe;
-		uint16_t tail;
 
 		cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
 		ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
@@ -2041,19 +2041,21 @@ mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq,
 			if (likely(ret != MLX5_CQE_STATUS_ERR)) {
 				/* No new CQEs in completion queue. */
 				assert(ret == MLX5_CQE_STATUS_HW_OWN);
-				if (likely(update)) {
-					/* Update the consumer index. */
-					rte_compiler_barrier();
-					*txq->cq_db =
-						rte_cpu_to_be_32(txq->cq_ci);
-				}
-				return;
+				break;
 			}
 			/* Some error occurred, try to restart. */
 			rte_wmb();
 			tail = mlx5_tx_error_cqe_handle
 				(txq, (volatile struct mlx5_err_cqe *)cqe);
+			if (likely(tail != txq->elts_tail)) {
+				mlx5_tx_free_elts(txq, tail, olx);
+				assert(tail == txq->elts_tail);
+			}
+			/* Allow flushing all CQEs from the queue. */
+			count = txq->cqe_s;
 		} else {
+			volatile struct mlx5_wqe_cseg *cseg;
+
 			/* Normal transmit completion. */
 			++txq->cq_ci;
 			rte_cio_rmb();
@@ -2066,13 +2068,27 @@ mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq,
 		if (txq->cq_pi)
 			--txq->cq_pi;
 #endif
-		if (likely(tail != txq->elts_tail)) {
-			/* Free data buffers from elts. */
-			mlx5_tx_free_elts(txq, tail, olx);
-			assert(tail == txq->elts_tail);
-		}
 		update = true;
-	} while (true);
+	/*
+	 * We have to restrict the amount of processed CQEs
+	 * in one tx_burst routine call. The CQ may be large
+	 * and many CQEs may be updated by the NIC in one
+	 * transaction. Buffers freeing is time consuming,
+	 * multiple iterations may introduce significant
+	 * latency.
+	 */
+	} while (--count);
+	if (likely(tail != txq->elts_tail)) {
+		/* Free data buffers from elts. */
+		mlx5_tx_free_elts(txq, tail, olx);
+		assert(tail == txq->elts_tail);
+	}
+	if (likely(update)) {
+		/* Update the consumer index. */
+		rte_compiler_barrier();
+		*txq->cq_db =
+		rte_cpu_to_be_32(txq->cq_ci);
+	}
 }
 
 /**
-- 
2.20.1