net/ena: support fast mbuf free

Add support for RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE offload. It can be
enabled if all the mbufs for a given queue belong to the same mempool
and their reference count is equal to 1.

Signed-off-by: Dawid Gorecki <dgr@semihalf.com>
Reviewed-by: Michal Krawczyk <mk@semihalf.com>
Reviewed-by: Shai Brandes <shaibran@amazon.com>
Reviewed-by: Amit Bernstein <amitbern@amazon.com>

diff --git a/doc/guides/nics/features/ena.ini b/doc/guides/nics/features/ena.ini
index 59c1ae85faf1640a9fe7f38d23430de96a43d0c5..1fe7a71e3d0043882149c2b41e76b8df6bcf4d5e 100644 (file)
--- a/doc/guides/nics/features/ena.ini
+++ b/doc/guides/nics/features/ena.ini
@@ -7,6 +7,7 @@
 Link status          = Y
 Link status event    = Y
 Rx interrupt         = Y
+Fast mbuf free       = Y
 Free Tx mbuf on demand = Y
 MTU update           = Y
 Scattered Rx         = Y

diff --git a/doc/guides/rel_notes/release_22_07.rst b/doc/guides/rel_notes/release_22_07.rst
index 847b8fffaab490c8a16e24ee1dc33bfc663b70a9..c2c4b957bb8b8086132cc6e3724511a1470767ab 100644 (file)
--- a/doc/guides/rel_notes/release_22_07.rst
+++ b/doc/guides/rel_notes/release_22_07.rst
@@ -102,6 +102,12 @@ New Features
 
   Added an API which can get the device type of vDPA device.
 
+* **Updated Amazon ena driver.**
+
+  The new driver version (v2.7.0) includes:
+
+  * Added fast mbuf free feature support.
+
 * **Updated Intel iavf driver.**
 
   * Added Tx QoS queue rate limitation support.

diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index 68768cab7077f8e0bab5067be821011d4019d580..68a4478410c92d1251177232cc554ad6a33a7d03 100644 (file)
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -36,6 +36,12 @@
 
 #define ENA_MIN_RING_DESC      128
 
+/*
+ * We should try to keep ENA_CLEANUP_BUF_SIZE lower than
+ * RTE_MEMPOOL_CACHE_MAX_SIZE, so we can fit this in mempool local cache.
+ */
+#define ENA_CLEANUP_BUF_SIZE   256
+
 #define ENA_PTYPE_HAS_HASH     (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP)
 
 struct ena_stats {
@@ -2402,6 +2408,8 @@ static uint64_t ena_get_tx_port_offloads(struct ena_adapter *adapter)
 
        port_offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
 
+       port_offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
+
        return port_offloads;
 }
 
@@ -2414,9 +2422,12 @@ static uint64_t ena_get_rx_queue_offloads(struct ena_adapter *adapter)
 
 static uint64_t ena_get_tx_queue_offloads(struct ena_adapter *adapter)
 {
+       uint64_t queue_offloads = 0;
        RTE_SET_USED(adapter);
 
-       return 0;
+       queue_offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
+
+       return queue_offloads;
 }
 
 static int ena_infos_get(struct rte_eth_dev *dev,
@@ -3001,13 +3012,38 @@ static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf)
        return 0;
 }
 
+static __rte_always_inline size_t
+ena_tx_cleanup_mbuf_fast(struct rte_mbuf **mbufs_to_clean,
+                        struct rte_mbuf *mbuf,
+                        size_t mbuf_cnt,
+                        size_t buf_size)
+{
+       struct rte_mbuf *m_next;
+
+       while (mbuf != NULL) {
+               m_next = mbuf->next;
+               mbufs_to_clean[mbuf_cnt++] = mbuf;
+               if (mbuf_cnt == buf_size) {
+                       rte_mempool_put_bulk(mbufs_to_clean[0]->pool, (void **)mbufs_to_clean,
+                               (unsigned int)mbuf_cnt);
+                       mbuf_cnt = 0;
+               }
+               mbuf = m_next;
+       }
+
+       return mbuf_cnt;
+}
+
 static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
 {
+       struct rte_mbuf *mbufs_to_clean[ENA_CLEANUP_BUF_SIZE];
        struct ena_ring *tx_ring = (struct ena_ring *)txp;
+       size_t mbuf_cnt = 0;
        unsigned int total_tx_descs = 0;
        unsigned int total_tx_pkts = 0;
        uint16_t cleanup_budget;
        uint16_t next_to_clean = tx_ring->next_to_clean;
+       bool fast_free = tx_ring->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
 
        /*
         * If free_pkt_cnt is equal to 0, it means that the user requested
@@ -3032,7 +3068,12 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
                tx_info->timestamp = 0;
 
                mbuf = tx_info->mbuf;
-               rte_pktmbuf_free(mbuf);
+               if (fast_free) {
+                       mbuf_cnt = ena_tx_cleanup_mbuf_fast(mbufs_to_clean, mbuf, mbuf_cnt,
+                               ENA_CLEANUP_BUF_SIZE);
+               } else {
+                       rte_pktmbuf_free(mbuf);
+               }
 
                tx_info->mbuf = NULL;
                tx_ring->empty_tx_reqs[next_to_clean] = req_id;
@@ -3052,6 +3093,10 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
                ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
        }
 
+       if (mbuf_cnt != 0)
+               rte_mempool_put_bulk(mbufs_to_clean[0]->pool,
+                       (void **)mbufs_to_clean, mbuf_cnt);
+
        /* Notify completion handler that full cleanup was performed */
        if (free_pkt_cnt == 0 || total_tx_pkts < cleanup_budget)
                tx_ring->last_cleanup_ticks = rte_get_timer_cycles();