net/mlx5: add fallback in Tx for multi-segment packet

mlx5_tx_burst_empw() falls back to legacy Tx descriptor for multi-segmented
packets without taking advantage of inlining. In many cases, the 1st
segment can be inlined and this could make device fetch only one segment
instead of two. This helps saving PCIe bandwidth when transmitting out
multi-segmented packets with still using the Enhanced Multi-Packet Send for
other packets.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 7c9d182709060dcee1e2aa652bdfb686dcbf1bec..d5e32b8454241b0d8c66aaf8a093f5bd4a57509c 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -1304,10 +1304,10 @@ mlx5_empw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
 }
 
 /**
- * DPDK callback for TX with Enhanced MPW support.
+ * TX with Enhanced MPW support.
  *
- * @param dpdk_txq
- *   Generic pointer to TX queue structure.
+ * @param txq
+ *   Pointer to TX queue structure.
  * @param[in] pkts
  *   Packets to transmit.
  * @param pkts_n
@@ -1316,10 +1316,10 @@ mlx5_empw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw)
  * @return
  *   Number of packets successfully transmitted (<= pkts_n).
  */
-uint16_t
-mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+static inline uint16_t
+txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
+              uint16_t pkts_n)
 {
-       struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
        uint16_t elts_head = txq->elts_head;
        const uint16_t elts_n = 1 << txq->elts_n;
        const uint16_t elts_m = elts_n - 1;
@@ -1584,6 +1584,47 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
        return i;
 }
 
+/**
+ * DPDK callback for TX with Enhanced MPW support.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+       struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
+       uint16_t nb_tx = 0;
+
+       while (pkts_n > nb_tx) {
+               uint16_t n;
+               uint16_t ret;
+
+               n = txq_count_contig_multi_seg(&pkts[nb_tx], pkts_n - nb_tx);
+               if (n) {
+                       ret = mlx5_tx_burst(dpdk_txq, &pkts[nb_tx], n);
+                       if (!ret)
+                               break;
+                       nb_tx += ret;
+               }
+               n = txq_count_contig_single_seg(&pkts[nb_tx], pkts_n - nb_tx);
+               if (n) {
+                       ret = txq_burst_empw(txq, &pkts[nb_tx], n);
+                       if (!ret)
+                               break;
+                       nb_tx += ret;
+               }
+       }
+       return nb_tx;
+}
+
 /**
  * Translate RX completion flags to packet type.
  *

diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index c072169f2041384e4bd4d08f3b601e1a412a0976..e70d52361abe925b0e12be5bd65565088b3badd9 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -652,4 +652,54 @@ txq_ol_cksum_to_cs(struct mlx5_txq_data *txq_data, struct rte_mbuf *buf)
        return cs_flags;
 }
 
+/**
+ * Count the number of contiguous single segment packets.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of contiguous single segment packets.
+ */
+static __rte_always_inline unsigned int
+txq_count_contig_single_seg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+       unsigned int pos;
+
+       if (!pkts_n)
+               return 0;
+       /* Count the number of contiguous single segment packets. */
+       for (pos = 0; pos < pkts_n; ++pos)
+               if (NB_SEGS(pkts[pos]) > 1)
+                       break;
+       return pos;
+}
+
+/**
+ * Count the number of contiguous multi-segment packets.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of contiguous multi-segment packets.
+ */
+static __rte_always_inline unsigned int
+txq_count_contig_multi_seg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+       unsigned int pos;
+
+       if (!pkts_n)
+               return 0;
+       /* Count the number of contiguous multi-segment packets. */
+       for (pos = 0; pos < pkts_n; ++pos)
+               if (NB_SEGS(pkts[pos]) == 1)
+                       break;
+       return pos;
+}
+
 #endif /* RTE_PMD_MLX5_RXTX_H_ */

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index 3aca17cb46a41f646bcb26c99d09e83b347f1652..8d23dae7e21c9ecdb4177792d5d11ad006b6b313 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -67,31 +67,6 @@
 #error "This should not be compiled if SIMD instructions are not supported."
 #endif
 
-/**
- * Count the number of continuous single segment packets.
- *
- * @param pkts
- *   Pointer to array of packets.
- * @param pkts_n
- *   Number of packets.
- *
- * @return
- *   Number of continuous single segment packets.
- */
-static inline unsigned int
-txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n)
-{
-       unsigned int pos;
-
-       if (!pkts_n)
-               return 0;
-       /* Count the number of continuous single segment packets. */
-       for (pos = 0; pos < pkts_n; ++pos)
-               if (NB_SEGS(pkts[pos]) > 1)
-                       break;
-       return pos;
-}
-
 /**
  * Count the number of packets having same ol_flags and calculate cs_flags.
  *
@@ -192,7 +167,7 @@ mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                                               pkts_n - nb_tx);
                n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
                if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
-                       n = txq_check_multiseg(&pkts[nb_tx], n);
+                       n = txq_count_contig_single_seg(&pkts[nb_tx], n);
                if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
                        n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
                ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index 661fbf11cfea1a5946d0045ad9483be6c2a32d37..c5d5b051908319ffa9570f766cef811631c95243 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -222,7 +222,7 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
  * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
  * it returns to make it processed by txq_scatter_v(). All the packets in
  * the pkts list should be single segment packets having same offload flags.
- * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ * This must be checked by txq_count_contig_single_seg() and txq_calc_offload().
  *
  * @param txq
  *   Pointer to TX queue structure.

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index 2b0e62861286b473de559c62134bd77c8ffedf08..0dd8145bc88438cf8367e12ed0d57e7aa84a0610 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -223,7 +223,7 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
  * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
  * it returns to make it processed by txq_scatter_v(). All the packets in
  * the pkts list should be single segment packets having same offload flags.
- * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ * This must be checked by txq_count_contig_single_seg() and txq_calc_offload().
  *
  * @param txq
  *   Pointer to TX queue structure.