X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mempool%2Frte_mempool.c;h=103c015e57915680314356a341160e48a987c3e8;hb=7508be4ccead7f092695d5534ab50b3b824baaad;hp=7472f830c109b418bbfc6e2bb5574c3823cf0d85;hpb=efd785f99489478d14e1c222e243ecbd0b151507;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index 7472f830c1..103c015e57 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -1,37 +1,9 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * Copyright(c) 2016 6WIND S.A. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation. + * Copyright(c) 2016 6WIND S.A. */ +#include #include #include #include @@ -127,6 +99,28 @@ static unsigned optimize_object_size(unsigned obj_size) return new_obj_size * RTE_MEMPOOL_ALIGN; } +static int +find_min_pagesz(const struct rte_memseg_list *msl, void *arg) +{ + size_t *min = arg; + + if (msl->page_sz < *min) + *min = msl->page_sz; + + return 0; +} + +static size_t +get_min_page_size(void) +{ + size_t min_pagesz = SIZE_MAX; + + rte_memseg_list_walk(find_min_pagesz, &min_pagesz); + + return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz; +} + + static void mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova) { @@ -357,21 +351,17 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) * on error. */ int -rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, - phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, +rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { unsigned total_elt_sz; + unsigned int mp_capa_flags; unsigned i = 0; size_t off; struct rte_mempool_memhdr *memhdr; int ret; - /* Notify memory area to mempool */ - ret = rte_mempool_ops_register_memory_area(mp, vaddr, paddr, len); - if (ret != -ENOTSUP && ret < 0) - return ret; - /* create the internal ring if not already done */ if ((mp->flags & MEMPOOL_F_POOL_CREATED) == 0) { ret = rte_mempool_ops_alloc(mp); @@ -380,21 +370,25 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, mp->flags |= MEMPOOL_F_POOL_CREATED; } + /* Notify memory area to mempool */ + ret = rte_mempool_ops_register_memory_area(mp, vaddr, iova, len); + if (ret != -ENOTSUP && ret < 0) + return ret; + /* mempool is already populated */ if (mp->populated_size >= mp->size) return -ENOSPC; total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; - /* Detect pool area has sufficient space for elements */ - if (mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG) { - if (len < total_elt_sz * mp->size) { - RTE_LOG(ERR, MEMPOOL, - "pool area %" PRIx64 " not enough\n", - (uint64_t)len); - return -ENOSPC; - } - } + /* Get mempool capabilities */ + mp_capa_flags = 0; + ret = rte_mempool_ops_get_capabilities(mp, &mp_capa_flags); + if ((ret < 0) && (ret != -ENOTSUP)) + return ret; + + /* update mempool capabilities */ + mp->flags |= mp_capa_flags; memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0); if (memhdr == NULL) @@ -402,12 +396,12 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, memhdr->mp = mp; memhdr->addr = vaddr; - memhdr->iova = paddr; + memhdr->iova = iova; memhdr->len = len; memhdr->free_cb = free_cb; memhdr->opaque = opaque; - if (mp->flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS) + if (mp_capa_flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS) /* align object start address to a multiple of total_elt_sz */ off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz); else if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) @@ -417,11 +411,11 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, while (off + total_elt_sz <= len && mp->populated_size < mp->size) { off += mp->header_size; - if (paddr == RTE_BAD_PHYS_ADDR) + if (iova == RTE_BAD_IOVA) mempool_add_elem(mp, (char *)vaddr + off, - RTE_BAD_PHYS_ADDR); + RTE_BAD_IOVA); else - mempool_add_elem(mp, (char *)vaddr + off, paddr + off); + mempool_add_elem(mp, (char *)vaddr + off, iova + off); off += mp->elt_size + mp->trailer_size; i++; } @@ -435,12 +429,20 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, return i; } +int +rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, + phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque) +{ + return rte_mempool_populate_iova(mp, vaddr, paddr, len, free_cb, opaque); +} + /* Add objects in the pool, using a table of physical pages. Return the * number of objects added, or a negative value on error. */ int -rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, +rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { uint32_t i, n; @@ -452,18 +454,18 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, return -EEXIST; if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - return rte_mempool_populate_phys(mp, vaddr, RTE_BAD_PHYS_ADDR, + return rte_mempool_populate_iova(mp, vaddr, RTE_BAD_IOVA, pg_num * pg_sz, free_cb, opaque); for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) { /* populate with the largest group of contiguous pages */ for (n = 1; (i + n) < pg_num && - paddr[i + n - 1] + pg_sz == paddr[i + n]; n++) + iova[i + n - 1] + pg_sz == iova[i + n]; n++) ; - ret = rte_mempool_populate_phys(mp, vaddr + i * pg_sz, - paddr[i], n * pg_sz, free_cb, opaque); + ret = rte_mempool_populate_iova(mp, vaddr + i * pg_sz, + iova[i], n * pg_sz, free_cb, opaque); if (ret < 0) { rte_mempool_free_memchunks(mp); return ret; @@ -475,6 +477,15 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, return cnt; } +int +rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, + const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) +{ + return rte_mempool_populate_iova_tab(mp, vaddr, paddr, pg_num, pg_shift, + free_cb, opaque); +} + /* Populate the mempool with a virtual area. Return the number of * objects added, or a negative value on error. */ @@ -497,7 +508,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, return -EINVAL; if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - return rte_mempool_populate_phys(mp, addr, RTE_BAD_PHYS_ADDR, + return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, len, free_cb, opaque); for (off = 0; off + pg_sz <= len && @@ -520,7 +531,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, break; } - ret = rte_mempool_populate_phys(mp, addr + off, iova, + ret = rte_mempool_populate_iova(mp, addr + off, iova, phys_len, free_cb, opaque); if (ret < 0) goto fail; @@ -551,6 +562,7 @@ rte_mempool_populate_default(struct rte_mempool *mp) unsigned mz_id, n; unsigned int mp_flags; int ret; + bool force_contig, no_contig, try_contig, no_pageshift; /* mempool must not be populated */ if (mp->nb_mem_chunks != 0) @@ -565,9 +577,68 @@ rte_mempool_populate_default(struct rte_mempool *mp) /* update mempool capabilities */ mp->flags |= mp_flags; - if (rte_eal_has_hugepages()) { - pg_shift = 0; /* not needed, zone is physically contiguous */ + no_contig = mp->flags & MEMPOOL_F_NO_PHYS_CONTIG; + force_contig = mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG; + + /* + * the following section calculates page shift and page size values. + * + * these values impact the result of rte_mempool_xmem_size(), which + * returns the amount of memory that should be allocated to store the + * desired number of objects. when not zero, it allocates more memory + * for the padding between objects, to ensure that an object does not + * cross a page boundary. in other words, page size/shift are to be set + * to zero if mempool elements won't care about page boundaries. + * there are several considerations for page size and page shift here. + * + * if we don't need our mempools to have physically contiguous objects, + * then just set page shift and page size to 0, because the user has + * indicated that there's no need to care about anything. + * + * if we do need contiguous objects, there is also an option to reserve + * the entire mempool memory as one contiguous block of memory, in + * which case the page shift and alignment wouldn't matter as well. + * + * if we require contiguous objects, but not necessarily the entire + * mempool reserved space to be contiguous, then there are two options. + * + * if our IO addresses are virtual, not actual physical (IOVA as VA + * case), then no page shift needed - our memory allocation will give us + * contiguous physical memory as far as the hardware is concerned, so + * act as if we're getting contiguous memory. + * + * if our IO addresses are physical, we may get memory from bigger + * pages, or we might get memory from smaller pages, and how much of it + * we require depends on whether we want bigger or smaller pages. + * However, requesting each and every memory size is too much work, so + * what we'll do instead is walk through the page sizes available, pick + * the smallest one and set up page shift to match that one. We will be + * wasting some space this way, but it's much nicer than looping around + * trying to reserve each and every page size. + * + * However, since size calculation will produce page-aligned sizes, it + * makes sense to first try and see if we can reserve the entire memzone + * in one contiguous chunk as well (otherwise we might end up wasting a + * 1G page on a 10MB memzone). If we fail to get enough contiguous + * memory, then we'll go and reserve space page-by-page. + */ + no_pageshift = no_contig || force_contig || + rte_eal_iova_mode() == RTE_IOVA_VA; + try_contig = !no_contig && !no_pageshift && rte_eal_has_hugepages(); + if (force_contig) + mz_flags |= RTE_MEMZONE_IOVA_CONTIG; + + if (no_pageshift) { pg_sz = 0; + pg_shift = 0; + align = RTE_CACHE_LINE_SIZE; + } else if (try_contig) { + pg_sz = get_min_page_size(); + pg_shift = rte_bsf32(pg_sz); + /* we're trying to reserve contiguous memzone first, so try + * align to cache line; if we fail to reserve a contiguous + * memzone, we'll adjust alignment to equal pagesize later. + */ align = RTE_CACHE_LINE_SIZE; } else { pg_sz = getpagesize(); @@ -577,8 +648,13 @@ rte_mempool_populate_default(struct rte_mempool *mp) total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { - size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift, - mp->flags); + unsigned int flags; + if (try_contig || no_pageshift) + size = rte_mempool_xmem_size(n, total_elt_sz, 0, + mp->flags); + else + size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift, + mp->flags); ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id); @@ -587,24 +663,53 @@ rte_mempool_populate_default(struct rte_mempool *mp) goto fail; } - mz = rte_memzone_reserve_aligned(mz_name, size, - mp->socket_id, mz_flags, align); - /* not enough memory, retry with the biggest zone we have */ - if (mz == NULL) + flags = mz_flags; + + /* if we're trying to reserve contiguous memory, add appropriate + * memzone flag. + */ + if (try_contig) + flags |= RTE_MEMZONE_IOVA_CONTIG; + + mz = rte_memzone_reserve_aligned(mz_name, size, mp->socket_id, + flags, align); + + /* if we were trying to allocate contiguous memory, adjust + * memzone size and page size to fit smaller page sizes, and + * try again. + */ + if (mz == NULL && try_contig) { + try_contig = false; + flags &= ~RTE_MEMZONE_IOVA_CONTIG; + align = pg_sz; + size = rte_mempool_xmem_size(n, total_elt_sz, + pg_shift, mp->flags); + + mz = rte_memzone_reserve_aligned(mz_name, size, + mp->socket_id, flags, align); + } + /* don't try reserving with 0 size if we were asked to reserve + * IOVA-contiguous memory. + */ + if (!force_contig && mz == NULL) { + /* not enough memory, retry with the biggest zone we + * have + */ mz = rte_memzone_reserve_aligned(mz_name, 0, - mp->socket_id, mz_flags, align); + mp->socket_id, flags, align); + } if (mz == NULL) { ret = -rte_errno; goto fail; } - if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + if (no_contig) iova = RTE_BAD_IOVA; else iova = mz->iova; - if (rte_eal_has_hugepages()) - ret = rte_mempool_populate_phys(mp, mz->addr, + if (no_pageshift || try_contig) + ret = rte_mempool_populate_iova(mp, mz->addr, iova, mz->len, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); @@ -990,7 +1095,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, if (mp_init) mp_init(mp, mp_init_arg); - ret = rte_mempool_populate_phys_tab(mp, vaddr, iova, pg_num, pg_shift, + ret = rte_mempool_populate_iova_tab(mp, vaddr, iova, pg_num, pg_shift, NULL, NULL); if (ret < 0 || ret != (int)mp->size) goto fail;