net/iavf: use write combining store for tail updates

Performance improvement: use a write combining store
instead of a regular mmio write to update queue tail
registers.

Signed-off-by: Gordon Noonan <gordon.noonan@intel.com>
Signed-off-by: Radu Nicolau <radu.nicolau@intel.com>
Acked-by: Qi Zhang <qi.z.zhang@intel.com>

diff --git a/drivers/common/iavf/iavf_osdep.h b/drivers/common/iavf/iavf_osdep.h
index 76c186a5b8c274255f6635dc671c7dccc2baa9bb..31d3d809f92622dc0504d68f7e8eaa2898d40ccd 100644 (file)
--- a/drivers/common/iavf/iavf_osdep.h
+++ b/drivers/common/iavf/iavf_osdep.h
@@ -132,6 +132,11 @@ writeq(uint64_t value, volatile void *addr)
 #define IAVF_PCI_REG_WRITE(reg, value)         writel(value, reg)
 #define IAVF_PCI_REG_WRITE_RELAXED(reg, value) writel_relaxed(value, reg)
 
+#define IAVF_PCI_REG_WC_WRITE(reg, value) \
+       rte_write32_wc((rte_cpu_to_le_32(value)), reg)
+#define IAVF_PCI_REG_WC_WRITE_RELAXED(reg, value) \
+       rte_write32_wc_relaxed((rte_cpu_to_le_32(value)), reg)
+
 #define IAVF_READ_REG(hw, reg)                 rd32(hw, reg)
 #define IAVF_WRITE_REG(hw, reg, value)         wr32(hw, reg, value)
 

diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index 6a713df828af2f95aa4584c4ab8cdc0a1bdb7886..fdab80a79df35435dd6210f9795a6ebbdd2764c0 100644 (file)
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -1161,7 +1161,7 @@ iavf_update_rx_tail(struct iavf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id)
                           rxq->port_id, rxq->queue_id, rx_id, nb_hold);
                rx_id = (uint16_t)((rx_id == 0) ?
                        (rxq->nb_rx_desc - 1) : (rx_id - 1));
-               IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+               IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
                nb_hold = 0;
        }
        rxq->nb_rx_hold = nb_hold;
@@ -1906,7 +1906,7 @@ iavf_rx_alloc_bufs(struct iavf_rx_queue *rxq)
 
        /* Update rx tail register */
        rte_wmb();
-       IAVF_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger);
+       IAVF_PCI_REG_WC_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger);
 
        rxq->rx_free_trigger =
                (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
@@ -2332,7 +2332,7 @@ end_of_tx:
        PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
                   txq->port_id, txq->queue_id, tx_id, nb_tx);
 
-       IAVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id);
+       IAVF_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id);
        txq->tx_tail = tx_id;
 
        return nb_tx;

diff --git a/drivers/net/iavf/iavf_rxtx_vec_avx2.c b/drivers/net/iavf/iavf_rxtx_vec_avx2.c
index f5646d645366f90a00ed9f8083be8c076298a33f..475070e036ef5e98bec0cbe086d2ff9986c5b2ae 100644 (file)
--- a/drivers/net/iavf/iavf_rxtx_vec_avx2.c
+++ b/drivers/net/iavf/iavf_rxtx_vec_avx2.c
@@ -1500,7 +1500,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
 
        txq->tx_tail = tx_id;
 
-       IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
+       IAVF_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
 
        return nb_pkts;
 }

diff --git a/drivers/net/iavf/iavf_rxtx_vec_avx512.c b/drivers/net/iavf/iavf_rxtx_vec_avx512.c
index d99de2a8bc5a7884abddb7df813eb6e1be41ce5e..571161c0cdecd26155368d3f1c66463df0d42aca 100644 (file)
--- a/drivers/net/iavf/iavf_rxtx_vec_avx512.c
+++ b/drivers/net/iavf/iavf_rxtx_vec_avx512.c
@@ -156,7 +156,7 @@ iavf_rxq_rearm(struct iavf_rx_queue *rxq)
                           (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
 
        /* Update the tail pointer on the NIC */
-       IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+       IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 }
 
 #define IAVF_RX_LEN_MASK 0x80808080
@@ -1958,7 +1958,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
 
        txq->tx_tail = tx_id;
 
-       IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
+       IAVF_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
 
        return nb_pkts;
 }

diff --git a/drivers/net/iavf/iavf_rxtx_vec_common.h b/drivers/net/iavf/iavf_rxtx_vec_common.h
index 6bb6903a8a968d79c942c6538b41a4bf091fb646..457d6339e1d7d0dbb943fe7076286a62254fb401 100644 (file)
--- a/drivers/net/iavf/iavf_rxtx_vec_common.h
+++ b/drivers/net/iavf/iavf_rxtx_vec_common.h
@@ -570,7 +570,7 @@ iavf_rxq_rearm_common(struct iavf_rx_queue *rxq, __rte_unused bool avx512)
                             (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
 
        /* Update the tail pointer on the NIC */
-       IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+       IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 }
 #endif
 

diff --git a/drivers/net/iavf/iavf_rxtx_vec_sse.c b/drivers/net/iavf/iavf_rxtx_vec_sse.c
index ef1f17ea214b57870949baf5d0987a0c85edf5e2..bf87696fa4cf4b56ba41678e1a38ab4b7fabfaf4 100644 (file)
--- a/drivers/net/iavf/iavf_rxtx_vec_sse.c
+++ b/drivers/net/iavf/iavf_rxtx_vec_sse.c
@@ -88,7 +88,7 @@ iavf_rxq_rearm(struct iavf_rx_queue *rxq)
                   rx_id, rxq->rxrearm_start, rxq->rxrearm_nb);
 
        /* Update the tail pointer on the NIC */
-       IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+       IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 }
 
 static inline void
@@ -1171,7 +1171,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
        PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_pkts=%u",
                   txq->port_id, txq->queue_id, tx_id, nb_pkts);
 
-       IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
+       IAVF_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
 
        return nb_pkts;
 }