mempool: fix slow allocation of large mempools

[dpdk.git] / lib / librte_mempool / rte_mempool.c

diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c
index 40cae3e..a8ad092 100644 (file)
--- a/lib/librte_mempool/rte_mempool.c
+++ b/lib/librte_mempool/rte_mempool.c
@@ -31,6 +31,7 @@
 #include <rte_string_fns.h>
 #include <rte_spinlock.h>
 #include <rte_tailq.h>
+#include <rte_function_versioning.h>
 
 #include "rte_mempool.h"
 
@@ -293,12 +294,17 @@ mempool_ops_alloc_once(struct rte_mempool *mp)
        return 0;
 }
 
+int
+rte_mempool_populate_iova_v20_0_1(struct rte_mempool *mp, char *vaddr,
+       rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
+       void *opaque);
+
 /* Add objects in the pool, using a physically contiguous memory
  * zone. Return the number of objects added, or a negative value
  * on error.
  */
 int
-rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
+rte_mempool_populate_iova_v20_0_1(struct rte_mempool *mp, char *vaddr,
        rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
        void *opaque)
 {
@@ -332,7 +338,7 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
                off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_MEMPOOL_ALIGN) - vaddr;
 
        if (off > len) {
-               ret = -EINVAL;
+               ret = -ENOBUFS;
                goto fail;
        }
 
@@ -343,7 +349,7 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
 
        /* not enough room to store one object */
        if (i == 0) {
-               ret = -EINVAL;
+               ret = -ENOBUFS;
                goto fail;
        }
 
@@ -355,6 +361,46 @@ fail:
        rte_free(memhdr);
        return ret;
 }
+BIND_DEFAULT_SYMBOL(rte_mempool_populate_iova, _v20_0_1, 20.0.1);
+MAP_STATIC_SYMBOL(
+       int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
+                               rte_iova_t iova, size_t len,
+                               rte_mempool_memchunk_free_cb_t *free_cb,
+                               void *opaque),
+       rte_mempool_populate_iova_v20_0_1);
+
+int
+rte_mempool_populate_iova_v20_0(struct rte_mempool *mp, char *vaddr,
+       rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
+       void *opaque);
+int
+rte_mempool_populate_iova_v20_0(struct rte_mempool *mp, char *vaddr,
+       rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
+       void *opaque)
+{
+       int ret;
+
+       ret = rte_mempool_populate_iova_v20_0_1(mp, vaddr, iova, len, free_cb,
+                                       opaque);
+       if (ret == -ENOBUFS)
+               ret = -EINVAL;
+
+       return ret;
+}
+VERSION_SYMBOL(rte_mempool_populate_iova, _v20_0, 20.0);
+
+static rte_iova_t
+get_iova(void *addr)
+{
+       struct rte_memseg *ms;
+
+       /* try registered memory first */
+       ms = rte_mem_virt2memseg(addr, NULL);
+       if (ms == NULL || ms->iova == RTE_BAD_IOVA)
+               /* fall back to actual physical address */
+               return rte_mem_virt2iova(addr);
+       return ms->iova + RTE_PTR_DIFF(addr, ms->addr);
+}
 
 /* Populate the mempool with a virtual area. Return the number of
  * objects added, or a negative value on error.
@@ -368,6 +414,13 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
        size_t off, phys_len;
        int ret, cnt = 0;
 
+       /* call alloc_once() in advance, it avoids a misinterpretation
+        * of -ENOBUFS when delegated to rte_mempool_populate_iova().
+        */
+       ret = mempool_ops_alloc_once(mp);
+       if (ret != 0)
+               return ret;
+
        if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG)
                return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA,
                        len, free_cb, opaque);
@@ -375,12 +428,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
        for (off = 0; off < len &&
                     mp->populated_size < mp->size; off += phys_len) {
 
-               iova = rte_mem_virt2iova(addr + off);
-
-               if (iova == RTE_BAD_IOVA && rte_eal_has_hugepages()) {
-                       ret = -EINVAL;
-                       goto fail;
-               }
+               iova = get_iova(addr + off);
 
                /* populate with the largest group of contiguous pages */
                for (phys_len = RTE_MIN(
@@ -391,15 +439,17 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
                     phys_len = RTE_MIN(phys_len + pg_sz, len - off)) {
                        rte_iova_t iova_tmp;
 
-                       iova_tmp = rte_mem_virt2iova(addr + off + phys_len);
+                       iova_tmp = get_iova(addr + off + phys_len);
 
                        if (iova_tmp == RTE_BAD_IOVA ||
                                        iova_tmp != iova + phys_len)
                                break;
                }
 
-               ret = rte_mempool_populate_iova(mp, addr + off, iova,
+               ret = rte_mempool_populate_iova_v20_0_1(mp, addr + off, iova,
                        phys_len, free_cb, opaque);
+               if (ret == -ENOBUFS)
+                       continue;
                if (ret < 0)
                        goto fail;
                /* no need to call the free callback for next chunks */
@@ -455,6 +505,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
        unsigned mz_id, n;
        int ret;
        bool need_iova_contig_obj;
+       size_t max_alloc_size = SIZE_MAX;
 
        ret = mempool_ops_alloc_once(mp);
        if (ret != 0)
@@ -534,30 +585,24 @@ rte_mempool_populate_default(struct rte_mempool *mp)
                if (min_chunk_size == (size_t)mem_size)
                        mz_flags |= RTE_MEMZONE_IOVA_CONTIG;
 
-               mz = rte_memzone_reserve_aligned(mz_name, mem_size,
+               /* Allocate a memzone, retrying with a smaller area on ENOMEM */
+               do {
+                       mz = rte_memzone_reserve_aligned(mz_name,
+                               RTE_MIN((size_t)mem_size, max_alloc_size),
                                mp->socket_id, mz_flags, align);
 
-               /* don't try reserving with 0 size if we were asked to reserve
-                * IOVA-contiguous memory.
-                */
-               if (min_chunk_size < (size_t)mem_size && mz == NULL) {
-                       /* not enough memory, retry with the biggest zone we
-                        * have
-                        */
-                       mz = rte_memzone_reserve_aligned(mz_name, 0,
-                                       mp->socket_id, mz_flags, align);
-               }
+                       if (mz == NULL && rte_errno != ENOMEM)
+                               break;
+
+                       max_alloc_size = RTE_MIN(max_alloc_size,
+                                               (size_t)mem_size) / 2;
+               } while (mz == NULL && max_alloc_size >= min_chunk_size);
+
                if (mz == NULL) {
                        ret = -rte_errno;
                        goto fail;
                }
 
-               if (mz->len < min_chunk_size) {
-                       rte_memzone_free(mz);
-                       ret = -ENOMEM;
-                       goto fail;
-               }
-
                if (need_iova_contig_obj)
                        iova = mz->iova;
                else