mbuf: fix mbuf free performance with non atomic refcnt

When RTE_MBUF_REFCNT_ATOMIC=n, the decrement of the mbuf reference
counter uses an atomic operation. This is not necessary and impacts
the performance (seen with TRex traffic generator).

We cannot replace rte_atomic16_add_return() by rte_mbuf_refcnt_update()
because it would add an additional check.

Solves this by introducing __rte_mbuf_refcnt_update(), which
updates the reference counter without doing anything else.

Fixes: 8f094a9ac5d7 ("mbuf: set mbuf fields while in pool")
Suggested-by: Hanoch Haim <hhaim@cisco.com>
Signed-off-by: Olivier Matz <olivier.matz@6wind.com>

diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index ce8a05d..dd08cb7 100644 (file)
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -764,6 +764,13 @@ rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
        rte_atomic16_set(&m->refcnt_atomic, new_value);
 }
 
+/* internal */
+static inline uint16_t
+__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
+{
+       return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
+}
+
 /**
  * Adds given value to an mbuf's refcnt and returns its new value.
  * @param m
@@ -788,19 +795,26 @@ rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
                return 1 + value;
        }
 
-       return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
+       return __rte_mbuf_refcnt_update(m, value);
 }
 
 #else /* ! RTE_MBUF_REFCNT_ATOMIC */
 
+/* internal */
+static inline uint16_t
+__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
+{
+       m->refcnt = (uint16_t)(m->refcnt + value);
+       return m->refcnt;
+}
+
 /**
  * Adds given value to an mbuf's refcnt and returns its new value.
  */
 static inline uint16_t
 rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
 {
-       m->refcnt = (uint16_t)(m->refcnt + value);
-       return m->refcnt;
+       return __rte_mbuf_refcnt_update(m, value);
 }
 
 /**
@@ -1364,8 +1378,7 @@ rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
 
                return m;
 
-       } else if (rte_atomic16_add_return(&m->refcnt_atomic, -1) == 0) {
-
+       } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
 
                if (RTE_MBUF_INDIRECT(m))
                        rte_pktmbuf_detach(m);