net/octeontx: support fast mbuf free

[dpdk.git] / lib / librte_mempool / rte_mempool.c

diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c
index aea5972..0be8f9f 100644 (file)
--- a/lib/librte_mempool/rte_mempool.c
+++ b/lib/librte_mempool/rte_mempool.c
@@ -33,6 +33,7 @@
 #include <rte_tailq.h>
 
 #include "rte_mempool.h"
+#include "rte_mempool_trace.h"
 
 TAILQ_HEAD(rte_mempool_list, rte_tailq_entry);
 
@@ -45,6 +46,7 @@ EAL_REGISTER_TAILQ(rte_mempool_tailq)
 #define CALC_CACHE_FLUSHTHRESH(c)      \
        ((typeof(c))((c) * CACHE_FLUSHTHRESH_MULTIPLIER))
 
+#if defined(RTE_ARCH_X86)
 /*
  * return the greatest common divisor between a and b (fast algorithm)
  *
@@ -74,12 +76,13 @@ static unsigned get_gcd(unsigned a, unsigned b)
 }
 
 /*
- * Depending on memory configuration, objects addresses are spread
+ * Depending on memory configuration on x86 arch, objects addresses are spread
  * between channels and ranks in RAM: the pool allocator will add
  * padding between objects. This function return the new size of the
  * object.
  */
-static unsigned optimize_object_size(unsigned obj_size)
+static unsigned int
+arch_mem_object_align(unsigned int obj_size)
 {
        unsigned nrank, nchan;
        unsigned new_obj_size;
@@ -99,6 +102,13 @@ static unsigned optimize_object_size(unsigned obj_size)
                new_obj_size++;
        return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
+#else
+static unsigned int
+arch_mem_object_align(unsigned int obj_size)
+{
+       return obj_size;
+}
+#endif
 
 struct pagesz_walk_arg {
        int socket_id;
@@ -234,8 +244,8 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
         */
        if ((flags & MEMPOOL_F_NO_SPREAD) == 0) {
                unsigned new_size;
-               new_size = optimize_object_size(sz->header_size + sz->elt_size +
-                       sz->trailer_size);
+               new_size = arch_mem_object_align
+                           (sz->header_size + sz->elt_size + sz->trailer_size);
                sz->trailer_size = new_size - sz->header_size - sz->elt_size;
        }
 
@@ -297,8 +307,8 @@ mempool_ops_alloc_once(struct rte_mempool *mp)
  * zone. Return the number of objects added, or a negative value
  * on error.
  */
-int
-rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
+static int
+__rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
        rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
        void *opaque)
 {
@@ -332,7 +342,7 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
                off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_MEMPOOL_ALIGN) - vaddr;
 
        if (off > len) {
-               ret = -EINVAL;
+               ret = 0;
                goto fail;
        }
 
@@ -343,7 +353,7 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
 
        /* not enough room to store one object */
        if (i == 0) {
-               ret = -EINVAL;
+               ret = 0;
                goto fail;
        }
 
@@ -356,6 +366,22 @@ fail:
        return ret;
 }
 
+int
+rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
+       rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
+       void *opaque)
+{
+       int ret;
+
+       ret = __rte_mempool_populate_iova(mp, vaddr, iova, len, free_cb,
+                                       opaque);
+       if (ret == 0)
+               ret = -EINVAL;
+
+       rte_mempool_trace_populate_iova(mp, vaddr, iova, len, free_cb, opaque);
+       return ret;
+}
+
 static rte_iova_t
 get_iova(void *addr)
 {
@@ -406,8 +432,10 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
                                break;
                }
 
-               ret = rte_mempool_populate_iova(mp, addr + off, iova,
+               ret = __rte_mempool_populate_iova(mp, addr + off, iova,
                        phys_len, free_cb, opaque);
+               if (ret == 0)
+                       continue;
                if (ret < 0)
                        goto fail;
                /* no need to call the free callback for next chunks */
@@ -415,6 +443,10 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
                cnt += ret;
        }
 
+       if (cnt == 0)
+               return -EINVAL;
+
+       rte_mempool_trace_populate_virt(mp, addr, len, pg_sz, free_cb, opaque);
        return cnt;
 
  fail:
@@ -444,6 +476,7 @@ rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz)
        else
                *pg_sz = getpagesize();
 
+       rte_mempool_trace_get_page_size(mp, *pg_sz);
        return 0;
 }
 
@@ -582,6 +615,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
                }
        }
 
+       rte_mempool_trace_populate_default(mp);
        return mp->size;
 
  fail:
@@ -672,6 +706,7 @@ rte_mempool_populate_anon(struct rte_mempool *mp)
                goto fail;
        }
 
+       rte_mempool_trace_populate_anon(mp);
        return mp->populated_size;
 
  fail:
@@ -703,6 +738,7 @@ rte_mempool_free(struct rte_mempool *mp)
        }
        rte_mcfg_tailq_write_unlock();
 
+       rte_mempool_trace_free(mp);
        rte_mempool_free_memchunks(mp);
        rte_mempool_ops_free(mp);
        rte_memzone_free(mp->mz);
@@ -741,6 +777,7 @@ rte_mempool_cache_create(uint32_t size, int socket_id)
 
        mempool_cache_init(cache, size);
 
+       rte_mempool_trace_cache_create(size, socket_id, cache);
        return cache;
 }
 
@@ -752,6 +789,7 @@ rte_mempool_cache_create(uint32_t size, int socket_id)
 void
 rte_mempool_cache_free(struct rte_mempool_cache *cache)
 {
+       rte_mempool_trace_cache_free(cache);
        rte_free(cache);
 }
 
@@ -882,6 +920,8 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
        rte_mcfg_tailq_write_unlock();
        rte_mcfg_mempool_write_unlock();
 
+       rte_mempool_trace_create_empty(name, n, elt_size, cache_size,
+               private_data_size, flags, mp);
        return mp;
 
 exit_unlock:
@@ -934,6 +974,9 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
        if (obj_init)
                rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
 
+       rte_mempool_trace_create(name, n, elt_size, cache_size,
+               private_data_size, mp_init, mp_init_arg, obj_init,
+               obj_init_arg, flags, mp);
        return mp;
 
  fail: