X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mempool%2Frte_mempool.c;h=213e574fc1a6ac48d90472e197462f1298276104;hb=035ee5bea5efde0b550bb3f1b5b4b6d35498b8e4;hp=238287a01550c89ebe569dc3567f3da1e85c007d;hpb=a36f5ce06eae4c49eb9bbeb64e64875250d5b028;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index 238287a015..213e574fc1 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "rte_mempool.h" @@ -367,17 +368,11 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t off, phys_len; int ret, cnt = 0; - /* address and len must be page-aligned */ - if (RTE_PTR_ALIGN_CEIL(addr, pg_sz) != addr) - return -EINVAL; - if (RTE_ALIGN_CEIL(len, pg_sz) != len) - return -EINVAL; - if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG) return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, len, free_cb, opaque); - for (off = 0; off + pg_sz <= len && + for (off = 0; off < len && mp->populated_size < mp->size; off += phys_len) { iova = rte_mem_virt2iova(addr + off); @@ -388,12 +383,18 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, } /* populate with the largest group of contiguous pages */ - for (phys_len = pg_sz; off + phys_len < len; phys_len += pg_sz) { + for (phys_len = RTE_MIN( + (size_t)(RTE_PTR_ALIGN_CEIL(addr + off + 1, pg_sz) - + (addr + off)), + len - off); + off + phys_len < len; + phys_len = RTE_MIN(phys_len + pg_sz, len - off)) { rte_iova_t iova_tmp; iova_tmp = rte_mem_virt2iova(addr + off + phys_len); - if (iova_tmp != iova + phys_len) + if (iova_tmp == RTE_BAD_IOVA || + iova_tmp != iova + phys_len) break; } @@ -428,24 +429,17 @@ rte_mempool_populate_default(struct rte_mempool *mp) rte_iova_t iova; unsigned mz_id, n; int ret; - bool no_contig, try_contig, no_pageshift, external; + bool need_iova_contig_obj; + bool alloc_in_ext_mem; ret = mempool_ops_alloc_once(mp); if (ret != 0) return ret; - /* check if we can retrieve a valid socket ID */ - ret = rte_malloc_heap_socket_is_external(mp->socket_id); - if (ret < 0) - return -EINVAL; - external = ret; - /* mempool must not be populated */ if (mp->nb_mem_chunks != 0) return -EEXIST; - no_contig = mp->flags & MEMPOOL_F_NO_IOVA_CONTIG; - /* * the following section calculates page shift and page size values. * @@ -482,29 +476,29 @@ rte_mempool_populate_default(struct rte_mempool *mp) * wasting some space this way, but it's much nicer than looping around * trying to reserve each and every page size. * - * However, since size calculation will produce page-aligned sizes, it - * makes sense to first try and see if we can reserve the entire memzone - * in one contiguous chunk as well (otherwise we might end up wasting a - * 1G page on a 10MB memzone). If we fail to get enough contiguous - * memory, then we'll go and reserve space page-by-page. + * If we fail to get enough contiguous memory, then we'll go and + * reserve space in smaller chunks. * * We also have to take into account the fact that memory that we're * going to allocate from can belong to an externally allocated memory * area, in which case the assumption of IOVA as VA mode being - * synonymous with IOVA contiguousness will not hold. We should also try - * to go for contiguous memory even if we're in no-huge mode, because - * external memory may in fact be IOVA-contiguous. + * synonymous with IOVA contiguousness will not hold. */ - external = rte_malloc_heap_socket_is_external(mp->socket_id) == 1; - no_pageshift = no_contig || - (!external && rte_eal_iova_mode() == RTE_IOVA_VA); - try_contig = !no_contig && !no_pageshift && - (rte_eal_has_hugepages() || external); - if (no_pageshift) { + /* check if we can retrieve a valid socket ID */ + ret = rte_malloc_heap_socket_is_external(mp->socket_id); + if (ret < 0) + return -EINVAL; + alloc_in_ext_mem = (ret == 1); + need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG); + + if (!need_iova_contig_obj) { + pg_sz = 0; + pg_shift = 0; + } else if (!alloc_in_ext_mem && rte_eal_iova_mode() == RTE_IOVA_VA) { pg_sz = 0; pg_shift = 0; - } else if (try_contig) { + } else if (rte_eal_has_hugepages() || alloc_in_ext_mem) { pg_sz = get_min_page_size(mp->socket_id); pg_shift = rte_bsf32(pg_sz); } else { @@ -514,14 +508,9 @@ rte_mempool_populate_default(struct rte_mempool *mp) for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { size_t min_chunk_size; - unsigned int flags; - if (try_contig || no_pageshift) - mem_size = rte_mempool_ops_calc_mem_size(mp, n, - 0, &min_chunk_size, &align); - else - mem_size = rte_mempool_ops_calc_mem_size(mp, n, - pg_shift, &min_chunk_size, &align); + mem_size = rte_mempool_ops_calc_mem_size( + mp, n, pg_shift, &min_chunk_size, &align); if (mem_size < 0) { ret = mem_size; @@ -535,35 +524,15 @@ rte_mempool_populate_default(struct rte_mempool *mp) goto fail; } - flags = mz_flags; - /* if we're trying to reserve contiguous memory, add appropriate * memzone flag. */ - if (try_contig) - flags |= RTE_MEMZONE_IOVA_CONTIG; + if (min_chunk_size == (size_t)mem_size) + mz_flags |= RTE_MEMZONE_IOVA_CONTIG; mz = rte_memzone_reserve_aligned(mz_name, mem_size, - mp->socket_id, flags, align); + mp->socket_id, mz_flags, align); - /* if we were trying to allocate contiguous memory, failed and - * minimum required contiguous chunk fits minimum page, adjust - * memzone size to the page size, and try again. - */ - if (mz == NULL && try_contig && min_chunk_size <= pg_sz) { - try_contig = false; - flags &= ~RTE_MEMZONE_IOVA_CONTIG; - - mem_size = rte_mempool_ops_calc_mem_size(mp, n, - pg_shift, &min_chunk_size, &align); - if (mem_size < 0) { - ret = mem_size; - goto fail; - } - - mz = rte_memzone_reserve_aligned(mz_name, mem_size, - mp->socket_id, flags, align); - } /* don't try reserving with 0 size if we were asked to reserve * IOVA-contiguous memory. */ @@ -572,8 +541,7 @@ rte_mempool_populate_default(struct rte_mempool *mp) * have */ mz = rte_memzone_reserve_aligned(mz_name, 0, - mp->socket_id, flags, - RTE_MAX(pg_sz, align)); + mp->socket_id, mz_flags, align); } if (mz == NULL) { ret = -rte_errno; @@ -586,19 +554,19 @@ rte_mempool_populate_default(struct rte_mempool *mp) goto fail; } - if (no_contig) - iova = RTE_BAD_IOVA; - else + if (need_iova_contig_obj) iova = mz->iova; + else + iova = RTE_BAD_IOVA; - if (no_pageshift || try_contig) + if (pg_sz == 0 || (mz_flags & RTE_MEMZONE_IOVA_CONTIG)) ret = rte_mempool_populate_iova(mp, mz->addr, iova, mz->len, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); else ret = rte_mempool_populate_virt(mp, mz->addr, - RTE_ALIGN_FLOOR(mz->len, pg_sz), pg_sz, + mz->len, pg_sz, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); if (ret < 0) { @@ -830,7 +798,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, return NULL; } - rte_rwlock_write_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_write_lock(); /* * reserve a memory zone for this mempool: private data is @@ -901,12 +869,12 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, rte_mcfg_tailq_write_lock(); TAILQ_INSERT_TAIL(mempool_list, te, next); rte_mcfg_tailq_write_unlock(); - rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_write_unlock(); return mp; exit_unlock: - rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_write_unlock(); rte_free(te); rte_mempool_free(mp); return NULL; @@ -1268,14 +1236,14 @@ rte_mempool_list_dump(FILE *f) mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_lock(); TAILQ_FOREACH(te, mempool_list, next) { mp = (struct rte_mempool *) te->data; rte_mempool_dump(f, mp); } - rte_rwlock_read_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_unlock(); } /* search a mempool from its name */ @@ -1288,7 +1256,7 @@ rte_mempool_lookup(const char *name) mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_lock(); TAILQ_FOREACH(te, mempool_list, next) { mp = (struct rte_mempool *) te->data; @@ -1296,7 +1264,7 @@ rte_mempool_lookup(const char *name) break; } - rte_rwlock_read_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_unlock(); if (te == NULL) { rte_errno = ENOENT; @@ -1315,11 +1283,11 @@ void rte_mempool_walk(void (*func)(struct rte_mempool *, void *), mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_lock(); TAILQ_FOREACH_SAFE(te, mempool_list, next, tmp_te) { (*func)((struct rte_mempool *) te->data, arg); } - rte_rwlock_read_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_unlock(); }