From 369f6077c515e52f393e5b6e3226453d9f0ec166 Mon Sep 17 00:00:00 2001 From: Lance Richardson Date: Tue, 6 Oct 2020 13:37:52 -0400 Subject: [PATCH] net/bnxt: support fast mbuf free Add support for DEV_TX_OFFLOAD_MBUF_FAST_FREE to bnxt vector mode transmit. This offload may be enabled only when multi-segment transmit is not needed, all transmitted mbufs for a given queue will be allocated from the same pool, and all transmitted mbufs will have a reference count of 1. Signed-off-by: Lance Richardson Reviewed-by: Ajit Khaparde --- doc/guides/nics/features/bnxt.ini | 1 + drivers/net/bnxt/bnxt_ethdev.c | 11 ++++--- drivers/net/bnxt/bnxt_rxtx_vec_common.h | 32 ++++++++++++++++++++ drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 38 ++++-------------------- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 39 ++++--------------------- drivers/net/bnxt/bnxt_txq.c | 2 ++ drivers/net/bnxt/bnxt_txq.h | 1 + drivers/net/bnxt/bnxt_txr.c | 5 +++- drivers/net/bnxt/bnxt_txr.h | 27 +++++++++++++++++ 9 files changed, 84 insertions(+), 72 deletions(-) diff --git a/doc/guides/nics/features/bnxt.ini b/doc/guides/nics/features/bnxt.ini index f1f300c8b2..e75cfc44dc 100644 --- a/doc/guides/nics/features/bnxt.ini +++ b/doc/guides/nics/features/bnxt.ini @@ -8,6 +8,7 @@ Speed capabilities = Y Link status = Y Link status event = Y Rx interrupt = Y +Fast mbuf free = Y Queue start/stop = Y Burst mode info = Y MTU update = Y diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 1bb0aa838a..8b63134c39 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -920,7 +920,9 @@ static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, dev_info->rx_offload_capa = BNXT_DEV_RX_OFFLOAD_SUPPORT; if (bp->flags & BNXT_FLAG_PTP_SUPPORTED) dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TIMESTAMP; - dev_info->tx_offload_capa = BNXT_DEV_TX_OFFLOAD_SUPPORT; + dev_info->tx_queue_offload_capa = DEV_TX_OFFLOAD_MBUF_FAST_FREE; + dev_info->tx_offload_capa = BNXT_DEV_TX_OFFLOAD_SUPPORT | + dev_info->tx_queue_offload_capa; dev_info->flow_type_rss_offloads = BNXT_ETH_RSS_SUPPORT; dev_info->speed_capa = bnxt_get_speed_capabilities(bp); @@ -1191,6 +1193,7 @@ bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) { #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) #ifndef RTE_LIBRTE_IEEE1588 + uint64_t offloads = eth_dev->data->dev_conf.txmode.offloads; struct bnxt *bp = eth_dev->data->dev_private; /* @@ -1198,7 +1201,7 @@ bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) * or tx offloads. */ if (!eth_dev->data->scattered_rx && - !eth_dev->data->dev_conf.txmode.offloads && + !(offloads & ~DEV_TX_OFFLOAD_MBUF_FAST_FREE) && !BNXT_TRUFLOW_EN(bp)) { PMD_DRV_LOG(INFO, "Using vector mode transmit for port %d\n", eth_dev->data->port_id); @@ -1210,7 +1213,7 @@ bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) "Port %d scatter: %d tx offload: %" PRIX64 "\n", eth_dev->data->port_id, eth_dev->data->scattered_rx, - eth_dev->data->dev_conf.txmode.offloads); + offloads); #endif #endif return bnxt_xmit_pkts; @@ -2685,7 +2688,7 @@ bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_free_thresh = txq->tx_free_thresh; qinfo->conf.tx_rs_thresh = 0; qinfo->conf.tx_deferred_start = txq->tx_deferred_start; - qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; + qinfo->conf.offloads = txq->offloads; } static const struct { diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h b/drivers/net/bnxt/bnxt_rxtx_vec_common.h index 4a48152fc1..33ac535682 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h +++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h @@ -94,4 +94,36 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) rxq->rxrearm_nb -= nb; } + +static inline void +bnxt_tx_cmp_vec(struct bnxt_tx_queue *txq, int nr_pkts) +{ + struct bnxt_tx_ring_info *txr = txq->tx_ring; + struct rte_mbuf **free = txq->free; + uint16_t cons = txr->tx_cons; + unsigned int blk = 0; + uint32_t ring_mask = txr->tx_ring_struct->ring_mask; + + while (nr_pkts--) { + struct bnxt_sw_tx_bd *tx_buf; + struct rte_mbuf *mbuf; + + tx_buf = &txr->tx_buf_ring[cons]; + cons = (cons + 1) & ring_mask; + mbuf = rte_pktmbuf_prefree_seg(tx_buf->mbuf); + if (unlikely(mbuf == NULL)) + continue; + tx_buf->mbuf = NULL; + + if (blk && mbuf->pool != free[0]->pool) { + rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); + blk = 0; + } + free[blk++] = mbuf; + } + if (blk) + rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); + + txr->tx_cons = cons; +} #endif /* _BNXT_RXTX_VEC_COMMON_H_ */ diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index 299b6b86df..4c04cc43a0 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -13,10 +13,10 @@ #include "bnxt.h" #include "bnxt_cpr.h" #include "bnxt_ring.h" -#include "bnxt_rxtx_vec_common.h" #include "bnxt_txq.h" #include "bnxt_txr.h" +#include "bnxt_rxtx_vec_common.h" /* * RX Ring handling @@ -338,37 +338,6 @@ out: return nb_rx_pkts; } -static void -bnxt_tx_cmp_vec(struct bnxt_tx_queue *txq, int nr_pkts) -{ - struct bnxt_tx_ring_info *txr = txq->tx_ring; - struct rte_mbuf **free = txq->free; - uint16_t cons = txr->tx_cons; - unsigned int blk = 0; - - while (nr_pkts--) { - struct bnxt_sw_tx_bd *tx_buf; - struct rte_mbuf *mbuf; - - tx_buf = &txr->tx_buf_ring[cons]; - cons = RING_NEXT(txr->tx_ring_struct, cons); - mbuf = rte_pktmbuf_prefree_seg(tx_buf->mbuf); - if (unlikely(mbuf == NULL)) - continue; - tx_buf->mbuf = NULL; - - if (blk && mbuf->pool != free[0]->pool) { - rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); - blk = 0; - } - free[blk++] = mbuf; - } - if (blk) - rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); - - txr->tx_cons = cons; -} - static void bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq) { @@ -399,7 +368,10 @@ bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq) cpr->valid = !!(raw_cons & cp_ring_struct->ring_size); if (nb_tx_pkts) { - bnxt_tx_cmp_vec(txq, nb_tx_pkts); + if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) + bnxt_tx_cmp_fast(txq, nb_tx_pkts); + else + bnxt_tx_cmp_vec(txq, nb_tx_pkts); cpr->cp_raw_cons = raw_cons; bnxt_db_cq(cpr); } diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 271e7908e3..f71f469852 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -13,10 +13,10 @@ #include "bnxt.h" #include "bnxt_cpr.h" #include "bnxt_ring.h" -#include "bnxt_rxtx_vec_common.h" #include "bnxt_txq.h" #include "bnxt_txr.h" +#include "bnxt_rxtx_vec_common.h" /* * RX Ring handling @@ -309,38 +309,6 @@ out: return nb_rx_pkts; } -static void -bnxt_tx_cmp_vec(struct bnxt_tx_queue *txq, int nr_pkts) -{ - struct bnxt_tx_ring_info *txr = txq->tx_ring; - struct rte_mbuf **free = txq->free; - uint16_t cons = txr->tx_cons; - unsigned int blk = 0; - uint32_t ring_mask = txr->tx_ring_struct->ring_mask; - - while (nr_pkts--) { - struct bnxt_sw_tx_bd *tx_buf; - struct rte_mbuf *mbuf; - - tx_buf = &txr->tx_buf_ring[cons]; - cons = (cons + 1) & ring_mask; - mbuf = rte_pktmbuf_prefree_seg(tx_buf->mbuf); - if (unlikely(mbuf == NULL)) - continue; - tx_buf->mbuf = NULL; - - if (blk && mbuf->pool != free[0]->pool) { - rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); - blk = 0; - } - free[blk++] = mbuf; - } - if (blk) - rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); - - txr->tx_cons = cons; -} - static void bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq) { @@ -371,7 +339,10 @@ bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq) cpr->valid = !!(raw_cons & cp_ring_struct->ring_size); if (nb_tx_pkts) { - bnxt_tx_cmp_vec(txq, nb_tx_pkts); + if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) + bnxt_tx_cmp_fast(txq, nb_tx_pkts); + else + bnxt_tx_cmp_vec(txq, nb_tx_pkts); cpr->cp_raw_cons = raw_cons; bnxt_db_cq(cpr); } diff --git a/drivers/net/bnxt/bnxt_txq.c b/drivers/net/bnxt/bnxt_txq.c index bdc7ffaaab..c8d75ac951 100644 --- a/drivers/net/bnxt/bnxt_txq.c +++ b/drivers/net/bnxt/bnxt_txq.c @@ -131,6 +131,8 @@ int bnxt_tx_queue_setup_op(struct rte_eth_dev *eth_dev, txq->nb_tx_desc = nb_desc; txq->tx_free_thresh = RTE_MIN(rte_align32pow2(nb_desc) / 4, RTE_BNXT_MAX_TX_BURST); + txq->offloads = eth_dev->data->dev_conf.txmode.offloads | + tx_conf->offloads; txq->tx_deferred_start = tx_conf->tx_deferred_start; diff --git a/drivers/net/bnxt/bnxt_txq.h b/drivers/net/bnxt/bnxt_txq.h index 9f849fc710..8033e241e9 100644 --- a/drivers/net/bnxt/bnxt_txq.h +++ b/drivers/net/bnxt/bnxt_txq.h @@ -39,6 +39,7 @@ struct bnxt_tx_queue { struct bnxt_cp_ring_info *cp_ring; const struct rte_memzone *mz; struct rte_mbuf **free; + uint64_t offloads; }; void bnxt_free_txq_stats(struct bnxt_tx_queue *txq); diff --git a/drivers/net/bnxt/bnxt_txr.c b/drivers/net/bnxt/bnxt_txr.c index c55497960b..125ac83416 100644 --- a/drivers/net/bnxt/bnxt_txr.c +++ b/drivers/net/bnxt/bnxt_txr.c @@ -428,7 +428,10 @@ static int bnxt_handle_tx_cp(struct bnxt_tx_queue *txq) } while (nb_tx_pkts < ring_mask); if (nb_tx_pkts) { - bnxt_tx_cmp(txq, nb_tx_pkts); + if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) + bnxt_tx_cmp_fast(txq, nb_tx_pkts); + else + bnxt_tx_cmp(txq, nb_tx_pkts); cpr->cp_raw_cons = raw_cons; bnxt_db_cq(cpr); } diff --git a/drivers/net/bnxt/bnxt_txr.h b/drivers/net/bnxt/bnxt_txr.h index 7715c11b83..026a65a723 100644 --- a/drivers/net/bnxt/bnxt_txr.h +++ b/drivers/net/bnxt/bnxt_txr.h @@ -52,6 +52,33 @@ static inline uint32_t bnxt_tx_avail(struct bnxt_tx_queue *txq) bnxt_tx_bds_in_hw(txq)) - 1); } +/* + * Transmit completion function for use when DEV_TX_OFFLOAD_MBUF_FAST_FREE + * is enabled. + */ +static inline void +bnxt_tx_cmp_fast(struct bnxt_tx_queue *txq, int nr_pkts) +{ + struct bnxt_tx_ring_info *txr = txq->tx_ring; + uint32_t ring_mask = txr->tx_ring_struct->ring_mask; + struct rte_mbuf **free = txq->free; + uint16_t cons = txr->tx_cons; + unsigned int blk = 0; + + while (nr_pkts--) { + struct bnxt_sw_tx_bd *tx_buf; + + tx_buf = &txr->tx_buf_ring[cons]; + cons = (cons + 1) & ring_mask; + free[blk++] = tx_buf->mbuf; + tx_buf->mbuf = NULL; + } + if (blk) + rte_mempool_put_bulk(free[0]->pool, (void **)free, blk); + + txr->tx_cons = cons; +} + void bnxt_free_tx_rings(struct bnxt *bp); int bnxt_init_one_tx_ring(struct bnxt_tx_queue *txq); int bnxt_init_tx_ring_struct(struct bnxt_tx_queue *txq, unsigned int socket_id); -- 2.20.1