net/octeontx: support fast mbuf free

[dpdk.git] / lib / librte_mempool / rte_mempool.c

diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c
index 08906df..0be8f9f 100644 (file)
--- a/lib/librte_mempool/rte_mempool.c
+++ b/lib/librte_mempool/rte_mempool.c
@@ -33,6 +33,7 @@
 #include <rte_tailq.h>
 
 #include "rte_mempool.h"
+#include "rte_mempool_trace.h"
 
 TAILQ_HEAD(rte_mempool_list, rte_tailq_entry);
 
@@ -45,6 +46,7 @@ EAL_REGISTER_TAILQ(rte_mempool_tailq)
 #define CALC_CACHE_FLUSHTHRESH(c)      \
        ((typeof(c))((c) * CACHE_FLUSHTHRESH_MULTIPLIER))
 
+#if defined(RTE_ARCH_X86)
 /*
  * return the greatest common divisor between a and b (fast algorithm)
  *
@@ -74,12 +76,13 @@ static unsigned get_gcd(unsigned a, unsigned b)
 }
 
 /*
- * Depending on memory configuration, objects addresses are spread
+ * Depending on memory configuration on x86 arch, objects addresses are spread
  * between channels and ranks in RAM: the pool allocator will add
  * padding between objects. This function return the new size of the
  * object.
  */
-static unsigned optimize_object_size(unsigned obj_size)
+static unsigned int
+arch_mem_object_align(unsigned int obj_size)
 {
        unsigned nrank, nchan;
        unsigned new_obj_size;
@@ -99,6 +102,13 @@ static unsigned optimize_object_size(unsigned obj_size)
                new_obj_size++;
        return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
+#else
+static unsigned int
+arch_mem_object_align(unsigned int obj_size)
+{
+       return obj_size;
+}
+#endif
 
 struct pagesz_walk_arg {
        int socket_id;
@@ -234,8 +244,8 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
         */
        if ((flags & MEMPOOL_F_NO_SPREAD) == 0) {
                unsigned new_size;
-               new_size = optimize_object_size(sz->header_size + sz->elt_size +
-                       sz->trailer_size);
+               new_size = arch_mem_object_align
+                           (sz->header_size + sz->elt_size + sz->trailer_size);
                sz->trailer_size = new_size - sz->header_size - sz->elt_size;
        }
 
@@ -368,6 +378,7 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
        if (ret == 0)
                ret = -EINVAL;
 
+       rte_mempool_trace_populate_iova(mp, vaddr, iova, len, free_cb, opaque);
        return ret;
 }
 
@@ -435,6 +446,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
        if (cnt == 0)
                return -EINVAL;
 
+       rte_mempool_trace_populate_virt(mp, addr, len, pg_sz, free_cb, opaque);
        return cnt;
 
  fail:
@@ -464,6 +476,7 @@ rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz)
        else
                *pg_sz = getpagesize();
 
+       rte_mempool_trace_get_page_size(mp, *pg_sz);
        return 0;
 }
 
@@ -602,6 +615,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
                }
        }
 
+       rte_mempool_trace_populate_default(mp);
        return mp->size;
 
  fail:
@@ -692,6 +706,7 @@ rte_mempool_populate_anon(struct rte_mempool *mp)
                goto fail;
        }
 
+       rte_mempool_trace_populate_anon(mp);
        return mp->populated_size;
 
  fail:
@@ -723,6 +738,7 @@ rte_mempool_free(struct rte_mempool *mp)
        }
        rte_mcfg_tailq_write_unlock();
 
+       rte_mempool_trace_free(mp);
        rte_mempool_free_memchunks(mp);
        rte_mempool_ops_free(mp);
        rte_memzone_free(mp->mz);
@@ -761,6 +777,7 @@ rte_mempool_cache_create(uint32_t size, int socket_id)
 
        mempool_cache_init(cache, size);
 
+       rte_mempool_trace_cache_create(size, socket_id, cache);
        return cache;
 }
 
@@ -772,6 +789,7 @@ rte_mempool_cache_create(uint32_t size, int socket_id)
 void
 rte_mempool_cache_free(struct rte_mempool_cache *cache)
 {
+       rte_mempool_trace_cache_free(cache);
        rte_free(cache);
 }
 
@@ -902,6 +920,8 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
        rte_mcfg_tailq_write_unlock();
        rte_mcfg_mempool_write_unlock();
 
+       rte_mempool_trace_create_empty(name, n, elt_size, cache_size,
+               private_data_size, flags, mp);
        return mp;
 
 exit_unlock:
@@ -954,6 +974,9 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
        if (obj_init)
                rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
 
+       rte_mempool_trace_create(name, n, elt_size, cache_size,
+               private_data_size, mp_init, mp_init_arg, obj_init,
+               obj_init_arg, flags, mp);
        return mp;
 
  fail: