X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mempool%2Frte_mempool.c;h=6fdb723ace6ae93c2a257ed3c98f54cb51eac654;hb=7e92cef514caab2e71f1f415375a1046c7a2377d;hp=df8a5278790054569db11c7ce55d03c0766fc3a3;hpb=8567eb37d9fcd7a54edab00d0abc06a2d2305baf;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index df8a527879..6fdb723ace 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -1,35 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * Copyright(c) 2016 6WIND S.A. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation. + * Copyright(c) 2016 6WIND S.A. */ #include @@ -40,6 +11,7 @@ #include #include #include +#include #include #include @@ -54,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -128,7 +99,7 @@ static unsigned optimize_object_size(unsigned obj_size) } static void -mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) +mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova) { struct rte_mempool_objhdr *hdr; struct rte_mempool_objtlr *tlr __rte_unused; @@ -136,7 +107,7 @@ mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) /* set mempool ptr in header */ hdr = RTE_PTR_SUB(obj, sizeof(*hdr)); hdr->mp = mp; - hdr->physaddr = physaddr; + hdr->iova = iova; STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next); mp->populated_size++; @@ -147,7 +118,7 @@ mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) #endif /* enqueue in ring */ - rte_ring_sp_enqueue(mp->ring, obj); + rte_mempool_ops_enqueue_bulk(mp, &obj, 1); } /* call obj_cb() for each mempool element */ @@ -198,7 +169,11 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, sz->header_size = RTE_ALIGN_CEIL(sz->header_size, RTE_MEMPOOL_ALIGN); +#ifdef RTE_LIBRTE_MEMPOOL_DEBUG sz->trailer_size = sizeof(struct rte_mempool_objtlr); +#else + sz->trailer_size = 0; +#endif /* element size is 8 bytes-aligned at least */ sz->elt_size = RTE_ALIGN_CEIL(elt_size, sizeof(uint64_t)); @@ -223,23 +198,6 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, sz->trailer_size = new_size - sz->header_size - sz->elt_size; } - if (! rte_eal_has_hugepages()) { - /* - * compute trailer size so that pool elements fit exactly in - * a standard page - */ - int page_size = getpagesize(); - int new_size = page_size - sz->header_size - sz->elt_size; - if (new_size < 0 || (unsigned int)new_size < sz->trailer_size) { - printf("When hugepages are disabled, pool objects " - "can't exceed PAGE_SIZE: %d + %d + %d > %d\n", - sz->header_size, sz->elt_size, sz->trailer_size, - page_size); - return 0; - } - sz->trailer_size = new_size; - } - /* this is the size of an object, including header and trailer */ sz->total_size = sz->header_size + sz->elt_size + sz->trailer_size; @@ -251,20 +209,30 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, * Calculate maximum amount of memory required to store given number of objects. */ size_t -rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift) +rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift, + unsigned int flags) { - size_t n, pg_num, pg_sz, sz; + size_t obj_per_page, pg_num, pg_sz; + unsigned int mask; - pg_sz = (size_t)1 << pg_shift; + mask = MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS | MEMPOOL_F_CAPA_PHYS_CONTIG; + if ((flags & mask) == mask) + /* alignment need one additional object */ + elt_num += 1; - if ((n = pg_sz / total_elt_sz) > 0) { - pg_num = (elt_num + n - 1) / n; - sz = pg_num << pg_shift; - } else { - sz = RTE_ALIGN_CEIL(total_elt_sz, pg_sz) * elt_num; - } + if (total_elt_sz == 0) + return 0; + + if (pg_shift == 0) + return total_elt_sz * elt_num; - return sz; + pg_sz = (size_t)1 << pg_shift; + obj_per_page = pg_sz / total_elt_sz; + if (obj_per_page == 0) + return RTE_ALIGN_CEIL(total_elt_sz, pg_sz) * elt_num; + + pg_num = (elt_num + obj_per_page - 1) / obj_per_page; + return pg_num << pg_shift; } /* @@ -273,23 +241,29 @@ rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift) */ ssize_t rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, - size_t total_elt_sz, const phys_addr_t paddr[], uint32_t pg_num, - uint32_t pg_shift) + size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num, + uint32_t pg_shift, unsigned int flags) { uint32_t elt_cnt = 0; - phys_addr_t start, end; - uint32_t paddr_idx; + rte_iova_t start, end; + uint32_t iova_idx; size_t pg_sz = (size_t)1 << pg_shift; + unsigned int mask; - /* if paddr is NULL, assume contiguous memory */ - if (paddr == NULL) { + mask = MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS | MEMPOOL_F_CAPA_PHYS_CONTIG; + if ((flags & mask) == mask) + /* alignment need one additional object */ + elt_num += 1; + + /* if iova is NULL, assume contiguous memory */ + if (iova == NULL) { start = 0; end = pg_sz * pg_num; - paddr_idx = pg_num; + iova_idx = pg_num; } else { - start = paddr[0]; - end = paddr[0] + pg_sz; - paddr_idx = 1; + start = iova[0]; + end = iova[0] + pg_sz; + iova_idx = 1; } while (elt_cnt < elt_num) { @@ -297,15 +271,15 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, /* enough contiguous memory, add an object */ start += total_elt_sz; elt_cnt++; - } else if (paddr_idx < pg_num) { + } else if (iova_idx < pg_num) { /* no room to store one obj, add a page */ - if (end == paddr[paddr_idx]) { + if (end == iova[iova_idx]) { end += pg_sz; } else { - start = paddr[paddr_idx]; - end = paddr[paddr_idx] + pg_sz; + start = iova[iova_idx]; + end = iova[iova_idx] + pg_sz; } - paddr_idx++; + iova_idx++; } else { /* no more page, return how many elements fit */ @@ -313,81 +287,16 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, } } - return (size_t)paddr_idx << pg_shift; + return (size_t)iova_idx << pg_shift; } -#ifndef RTE_LIBRTE_XEN_DOM0 -/* stub if DOM0 support not configured */ -struct rte_mempool * -rte_dom0_mempool_create(const char *name __rte_unused, - unsigned n __rte_unused, - unsigned elt_size __rte_unused, - unsigned cache_size __rte_unused, - unsigned private_data_size __rte_unused, - rte_mempool_ctor_t *mp_init __rte_unused, - void *mp_init_arg __rte_unused, - rte_mempool_obj_ctor_t *obj_init __rte_unused, - void *obj_init_arg __rte_unused, - int socket_id __rte_unused, - unsigned flags __rte_unused) -{ - rte_errno = EINVAL; - return NULL; -} -#endif - -/* create the mempool */ -struct rte_mempool * -rte_mempool_create(const char *name, unsigned n, unsigned elt_size, - unsigned cache_size, unsigned private_data_size, - rte_mempool_ctor_t *mp_init, void *mp_init_arg, - rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, - int socket_id, unsigned flags) -{ - if (rte_xen_dom0_supported()) - return rte_dom0_mempool_create(name, n, elt_size, - cache_size, private_data_size, - mp_init, mp_init_arg, - obj_init, obj_init_arg, - socket_id, flags); - else - return rte_mempool_xmem_create(name, n, elt_size, - cache_size, private_data_size, - mp_init, mp_init_arg, - obj_init, obj_init_arg, - socket_id, flags, - NULL, NULL, MEMPOOL_PG_NUM_DEFAULT, - MEMPOOL_PG_SHIFT_MAX); -} - -/* create the internal ring */ -static int -rte_mempool_ring_create(struct rte_mempool *mp) -{ - int rg_flags = 0; - char rg_name[RTE_RING_NAMESIZE]; - struct rte_ring *r; - - snprintf(rg_name, sizeof(rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name); - - /* ring flags */ - if (mp->flags & MEMPOOL_F_SP_PUT) - rg_flags |= RING_F_SP_ENQ; - if (mp->flags & MEMPOOL_F_SC_GET) - rg_flags |= RING_F_SC_DEQ; - - /* Allocate the ring that will be used to store objects. - * Ring functions will return appropriate errors if we are - * running as a secondary process etc., so no checks made - * in this function for that condition. - */ - r = rte_ring_create(rg_name, rte_align32pow2(mp->size + 1), - mp->socket_id, rg_flags); - if (r == NULL) - return -rte_errno; - - mp->ring = r; - return 0; +/* free a memchunk allocated with rte_memzone_reserve() */ +static void +rte_mempool_memchunk_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr, + void *opaque) +{ + const struct rte_memzone *mz = opaque; + rte_memzone_free(mz); } /* Free memory chunks used by a mempool. Objects must be in pool */ @@ -398,7 +307,7 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) void *elt; while (!STAILQ_EMPTY(&mp->elt_list)) { - rte_ring_sc_dequeue(mp->ring, &elt); + rte_mempool_ops_dequeue_bulk(mp, &elt, 1); (void)elt; STAILQ_REMOVE_HEAD(&mp->elt_list, next); mp->populated_size--; @@ -407,6 +316,8 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) while (!STAILQ_EMPTY(&mp->mem_list)) { memhdr = STAILQ_FIRST(&mp->mem_list); STAILQ_REMOVE_HEAD(&mp->mem_list, next); + if (memhdr->free_cb != NULL) + memhdr->free_cb(memhdr, memhdr->opaque); rte_free(memhdr); mp->nb_mem_chunks--; } @@ -416,14 +327,29 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) * zone. Return the number of objects added, or a negative value * on error. */ -static int -rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, - phys_addr_t paddr, size_t len) +int +rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque) { unsigned total_elt_sz; unsigned i = 0; size_t off; struct rte_mempool_memhdr *memhdr; + int ret; + + /* create the internal ring if not already done */ + if ((mp->flags & MEMPOOL_F_POOL_CREATED) == 0) { + ret = rte_mempool_ops_alloc(mp); + if (ret != 0) + return ret; + mp->flags |= MEMPOOL_F_POOL_CREATED; + } + + /* Notify memory area to mempool */ + ret = rte_mempool_ops_register_memory_area(mp, vaddr, iova, len); + if (ret != -ENOTSUP && ret < 0) + return ret; /* mempool is already populated */ if (mp->populated_size >= mp->size) @@ -431,23 +357,42 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + /* Detect pool area has sufficient space for elements */ + if (mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG) { + if (len < total_elt_sz * mp->size) { + RTE_LOG(ERR, MEMPOOL, + "pool area %" PRIx64 " not enough\n", + (uint64_t)len); + return -ENOSPC; + } + } + memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0); if (memhdr == NULL) return -ENOMEM; memhdr->mp = mp; memhdr->addr = vaddr; - memhdr->phys_addr = paddr; + memhdr->iova = iova; memhdr->len = len; + memhdr->free_cb = free_cb; + memhdr->opaque = opaque; - if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) + if (mp->flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS) + /* align object start address to a multiple of total_elt_sz */ + off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz); + else if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr; else off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_CACHE_LINE_SIZE) - vaddr; while (off + total_elt_sz <= len && mp->populated_size < mp->size) { off += mp->header_size; - mempool_add_elem(mp, (char *)vaddr + off, paddr + off); + if (iova == RTE_BAD_IOVA) + mempool_add_elem(mp, (char *)vaddr + off, + RTE_BAD_IOVA); + else + mempool_add_elem(mp, (char *)vaddr + off, iova + off); off += mp->elt_size + mp->trailer_size; i++; } @@ -461,12 +406,21 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, return i; } +int +rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, + phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque) +{ + return rte_mempool_populate_iova(mp, vaddr, paddr, len, free_cb, opaque); +} + /* Add objects in the pool, using a table of physical pages. Return the * number of objects added, or a negative value on error. */ -static int -rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift) +int +rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { uint32_t i, n; int ret, cnt = 0; @@ -476,50 +430,345 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, if (mp->nb_mem_chunks != 0) return -EEXIST; + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + return rte_mempool_populate_iova(mp, vaddr, RTE_BAD_IOVA, + pg_num * pg_sz, free_cb, opaque); + for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) { /* populate with the largest group of contiguous pages */ for (n = 1; (i + n) < pg_num && - paddr[i] + pg_sz == paddr[i+n]; n++) + iova[i + n - 1] + pg_sz == iova[i + n]; n++) ; - ret = rte_mempool_populate_phys(mp, vaddr + i * pg_sz, - paddr[i], n * pg_sz); + ret = rte_mempool_populate_iova(mp, vaddr + i * pg_sz, + iova[i], n * pg_sz, free_cb, opaque); if (ret < 0) { rte_mempool_free_memchunks(mp); return ret; } + /* no need to call the free callback for next chunks */ + free_cb = NULL; + cnt += ret; + } + return cnt; +} + +int +rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, + const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) +{ + return rte_mempool_populate_iova_tab(mp, vaddr, paddr, pg_num, pg_shift, + free_cb, opaque); +} + +/* Populate the mempool with a virtual area. Return the number of + * objects added, or a negative value on error. + */ +int +rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, + size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque) +{ + rte_iova_t iova; + size_t off, phys_len; + int ret, cnt = 0; + + /* mempool must not be populated */ + if (mp->nb_mem_chunks != 0) + return -EEXIST; + /* address and len must be page-aligned */ + if (RTE_PTR_ALIGN_CEIL(addr, pg_sz) != addr) + return -EINVAL; + if (RTE_ALIGN_CEIL(len, pg_sz) != len) + return -EINVAL; + + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, + len, free_cb, opaque); + + for (off = 0; off + pg_sz <= len && + mp->populated_size < mp->size; off += phys_len) { + + iova = rte_mem_virt2iova(addr + off); + + if (iova == RTE_BAD_IOVA && rte_eal_has_hugepages()) { + ret = -EINVAL; + goto fail; + } + + /* populate with the largest group of contiguous pages */ + for (phys_len = pg_sz; off + phys_len < len; phys_len += pg_sz) { + rte_iova_t iova_tmp; + + iova_tmp = rte_mem_virt2iova(addr + off + phys_len); + + if (iova_tmp != iova + phys_len) + break; + } + + ret = rte_mempool_populate_iova(mp, addr + off, iova, + phys_len, free_cb, opaque); + if (ret < 0) + goto fail; + /* no need to call the free callback for next chunks */ + free_cb = NULL; cnt += ret; } + return cnt; + + fail: + rte_mempool_free_memchunks(mp); + return ret; +} + +/* Default function to populate the mempool: allocate memory in memzones, + * and populate them. Return the number of objects added, or a negative + * value on error. + */ +int +rte_mempool_populate_default(struct rte_mempool *mp) +{ + unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; + char mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *mz; + size_t size, total_elt_sz, align, pg_sz, pg_shift; + rte_iova_t iova; + unsigned mz_id, n; + unsigned int mp_flags; + int ret; + + /* mempool must not be populated */ + if (mp->nb_mem_chunks != 0) + return -EEXIST; + + /* Get mempool capabilities */ + mp_flags = 0; + ret = rte_mempool_ops_get_capabilities(mp, &mp_flags); + if ((ret < 0) && (ret != -ENOTSUP)) + return ret; + + /* update mempool capabilities */ + mp->flags |= mp_flags; + + if (rte_eal_has_hugepages()) { + pg_shift = 0; /* not needed, zone is physically contiguous */ + pg_sz = 0; + align = RTE_CACHE_LINE_SIZE; + } else { + pg_sz = getpagesize(); + pg_shift = rte_bsf32(pg_sz); + align = pg_sz; + } + + total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { + size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift, + mp->flags); + + ret = snprintf(mz_name, sizeof(mz_name), + RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id); + if (ret < 0 || ret >= (int)sizeof(mz_name)) { + ret = -ENAMETOOLONG; + goto fail; + } + + mz = rte_memzone_reserve_aligned(mz_name, size, + mp->socket_id, mz_flags, align); + /* not enough memory, retry with the biggest zone we have */ + if (mz == NULL) + mz = rte_memzone_reserve_aligned(mz_name, 0, + mp->socket_id, mz_flags, align); + if (mz == NULL) { + ret = -rte_errno; + goto fail; + } + + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + iova = RTE_BAD_IOVA; + else + iova = mz->iova; + + if (rte_eal_has_hugepages()) + ret = rte_mempool_populate_iova(mp, mz->addr, + iova, mz->len, + rte_mempool_memchunk_mz_free, + (void *)(uintptr_t)mz); + else + ret = rte_mempool_populate_virt(mp, mz->addr, + mz->len, pg_sz, + rte_mempool_memchunk_mz_free, + (void *)(uintptr_t)mz); + if (ret < 0) { + rte_memzone_free(mz); + goto fail; + } + } + + return mp->size; + + fail: + rte_mempool_free_memchunks(mp); + return ret; +} + +/* return the memory size required for mempool objects in anonymous mem */ +static size_t +get_anon_size(const struct rte_mempool *mp) +{ + size_t size, total_elt_sz, pg_sz, pg_shift; + + pg_sz = getpagesize(); + pg_shift = rte_bsf32(pg_sz); + total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift, + mp->flags); + + return size; +} + +/* unmap a memory zone mapped by rte_mempool_populate_anon() */ +static void +rte_mempool_memchunk_anon_free(struct rte_mempool_memhdr *memhdr, + void *opaque) +{ + munmap(opaque, get_anon_size(memhdr->mp)); +} + +/* populate the mempool with an anonymous mapping */ +int +rte_mempool_populate_anon(struct rte_mempool *mp) +{ + size_t size; + int ret; + char *addr; + + /* mempool is already populated, error */ + if (!STAILQ_EMPTY(&mp->mem_list)) { + rte_errno = EINVAL; + return 0; + } + + /* get chunk of virtually continuous memory */ + size = get_anon_size(mp); + addr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + rte_errno = errno; + return 0; + } + /* can't use MMAP_LOCKED, it does not exist on BSD */ + if (mlock(addr, size) < 0) { + rte_errno = errno; + munmap(addr, size); + return 0; + } + + ret = rte_mempool_populate_virt(mp, addr, size, getpagesize(), + rte_mempool_memchunk_anon_free, addr); + if (ret == 0) + goto fail; + + return mp->populated_size; + + fail: + rte_mempool_free_memchunks(mp); + return 0; +} + +/* free a mempool */ +void +rte_mempool_free(struct rte_mempool *mp) +{ + struct rte_mempool_list *mempool_list = NULL; + struct rte_tailq_entry *te; + + if (mp == NULL) + return; + + mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + /* find out tailq entry */ + TAILQ_FOREACH(te, mempool_list, next) { + if (te->data == (void *)mp) + break; + } + + if (te != NULL) { + TAILQ_REMOVE(mempool_list, te, next); + rte_free(te); + } + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + + rte_mempool_free_memchunks(mp); + rte_mempool_ops_free(mp); + rte_memzone_free(mp->mz); +} + +static void +mempool_cache_init(struct rte_mempool_cache *cache, uint32_t size) +{ + cache->size = size; + cache->flushthresh = CALC_CACHE_FLUSHTHRESH(size); + cache->len = 0; } /* - * Create the mempool over already allocated chunk of memory. - * That external memory buffer can consists of physically disjoint pages. - * Setting vaddr to NULL, makes mempool to fallback to original behaviour - * and allocate space for mempool and it's elements as one big chunk of - * physically continuos memory. - * */ + * Create and initialize a cache for objects that are retrieved from and + * returned to an underlying mempool. This structure is identical to the + * local_cache[lcore_id] pointed to by the mempool structure. + */ +struct rte_mempool_cache * +rte_mempool_cache_create(uint32_t size, int socket_id) +{ + struct rte_mempool_cache *cache; + + if (size == 0 || size > RTE_MEMPOOL_CACHE_MAX_SIZE) { + rte_errno = EINVAL; + return NULL; + } + + cache = rte_zmalloc_socket("MEMPOOL_CACHE", sizeof(*cache), + RTE_CACHE_LINE_SIZE, socket_id); + if (cache == NULL) { + RTE_LOG(ERR, MEMPOOL, "Cannot allocate mempool cache.\n"); + rte_errno = ENOMEM; + return NULL; + } + + mempool_cache_init(cache, size); + + return cache; +} + +/* + * Free a cache. It's the responsibility of the user to make sure that any + * remaining objects in the cache are flushed to the corresponding + * mempool. + */ +void +rte_mempool_cache_free(struct rte_mempool_cache *cache) +{ + rte_free(cache); +} + +/* create an empty mempool */ struct rte_mempool * -rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, - unsigned cache_size, unsigned private_data_size, - rte_mempool_ctor_t *mp_init, void *mp_init_arg, - rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, - int socket_id, unsigned flags, void *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift) +rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, + unsigned cache_size, unsigned private_data_size, + int socket_id, unsigned flags) { char mz_name[RTE_MEMZONE_NAMESIZE]; struct rte_mempool_list *mempool_list; struct rte_mempool *mp = NULL; struct rte_tailq_entry *te = NULL; - const struct rte_memzone *mz; + const struct rte_memzone *mz = NULL; size_t mempool_size; - int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; - void *obj; + unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; struct rte_mempool_objsz objsz; - void *startaddr; - int page_size = getpagesize(); + unsigned lcore_id; int ret; /* compilation-time checks */ @@ -543,18 +792,6 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, return NULL; } - /* check that we have both VA and PA */ - if (vaddr != NULL && paddr == NULL) { - rte_errno = EINVAL; - return NULL; - } - - /* Check that pg_num and pg_shift parameters are valid. */ - if (pg_num == 0 || pg_shift > MEMPOOL_PG_SHIFT_MAX) { - rte_errno = EINVAL; - return NULL; - } - /* "no cache align" imply "no spread" */ if (flags & MEMPOOL_F_NO_CACHE_ALIGN) flags |= MEMPOOL_F_NO_SPREAD; @@ -574,16 +811,6 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, private_data_size = (private_data_size + RTE_MEMPOOL_ALIGN_MASK) & (~RTE_MEMPOOL_ALIGN_MASK); - if (! rte_eal_has_hugepages()) { - /* - * expand private data size to a whole page, so that the - * first pool element will start on a new standard page - */ - int head = sizeof(struct rte_mempool); - int new_size = (private_data_size + head) % page_size; - if (new_size) - private_data_size += page_size - new_size; - } /* try to allocate tailq entry */ te = rte_zmalloc("MEMPOOL_TAILQ_ENTRY", sizeof(*te), 0); @@ -592,64 +819,41 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, goto exit_unlock; } - /* - * If user provided an external memory buffer, then use it to - * store mempool objects. Otherwise reserve a memzone that is large - * enough to hold mempool header and metadata plus mempool objects. - */ mempool_size = MEMPOOL_HEADER_SIZE(mp, cache_size); mempool_size += private_data_size; mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN); - if (vaddr == NULL) - mempool_size += (size_t)objsz.total_size * n; - if (! rte_eal_has_hugepages()) { - /* - * we want the memory pool to start on a page boundary, - * because pool elements crossing page boundaries would - * result in discontiguous physical addresses - */ - mempool_size += page_size; + ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name); + if (ret < 0 || ret >= (int)sizeof(mz_name)) { + rte_errno = ENAMETOOLONG; + goto exit_unlock; } - snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name); - mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags); if (mz == NULL) goto exit_unlock; - if (rte_eal_has_hugepages()) { - startaddr = (void*)mz->addr; - } else { - /* align memory pool start address on a page boundary */ - unsigned long addr = (unsigned long)mz->addr; - if (addr & (page_size - 1)) { - addr += page_size; - addr &= ~(page_size - 1); - } - startaddr = (void*)addr; - } - /* init the mempool structure */ - mp = startaddr; - memset(mp, 0, sizeof(*mp)); - snprintf(mp->name, sizeof(mp->name), "%s", name); - mp->phys_addr = mz->phys_addr; - mp->socket_id = socket_id; + mp = mz->addr; + memset(mp, 0, MEMPOOL_HEADER_SIZE(mp, cache_size)); + ret = snprintf(mp->name, sizeof(mp->name), "%s", name); + if (ret < 0 || ret >= (int)sizeof(mp->name)) { + rte_errno = ENAMETOOLONG; + goto exit_unlock; + } + mp->mz = mz; mp->size = n; mp->flags = flags; + mp->socket_id = socket_id; mp->elt_size = objsz.elt_size; mp->header_size = objsz.header_size; mp->trailer_size = objsz.trailer_size; + /* Size of default caches, zero means disabled. */ mp->cache_size = cache_size; - mp->cache_flushthresh = CALC_CACHE_FLUSHTHRESH(cache_size); mp->private_data_size = private_data_size; STAILQ_INIT(&mp->elt_list); STAILQ_INIT(&mp->mem_list); - if (rte_mempool_ring_create(mp) < 0) - goto exit_unlock; - /* * local_cache pointer is set even if cache_size is zero. * The local_cache points to just past the elt_pa[] array. @@ -657,34 +861,14 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, mp->local_cache = (struct rte_mempool_cache *) RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, 0)); - /* call the initializer */ - if (mp_init) - mp_init(mp, mp_init_arg); - - /* mempool elements allocated together with mempool */ - if (vaddr == NULL) { - /* calculate address of the first elt for continuous mempool. */ - obj = (char *)mp + MEMPOOL_HEADER_SIZE(mp, cache_size) + - private_data_size; - obj = RTE_PTR_ALIGN_CEIL(obj, RTE_MEMPOOL_ALIGN); - - ret = rte_mempool_populate_phys(mp, obj, - mp->phys_addr + ((char *)obj - (char *)mp), - objsz.total_size * n); - if (ret != (int)mp->size) - goto exit_unlock; - } else { - ret = rte_mempool_populate_phys_tab(mp, vaddr, - paddr, pg_num, pg_shift); - if (ret != (int)mp->size) - goto exit_unlock; + /* Init all default caches. */ + if (cache_size != 0) { + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) + mempool_cache_init(&mp->local_cache[lcore_id], + cache_size); } - /* call the initializer */ - if (obj_init) - rte_mempool_obj_iter(mp, obj_init, obj_init_arg); - - te->data = (void *) mp; + te->data = mp; rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); TAILQ_INSERT_TAIL(mempool_list, te, next); @@ -695,23 +879,129 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, exit_unlock: rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK); - if (mp != NULL) { - rte_mempool_free_memchunks(mp); - rte_ring_free(mp->ring); - } rte_free(te); + rte_mempool_free(mp); + return NULL; +} + +/* create the mempool */ +struct rte_mempool * +rte_mempool_create(const char *name, unsigned n, unsigned elt_size, + unsigned cache_size, unsigned private_data_size, + rte_mempool_ctor_t *mp_init, void *mp_init_arg, + rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, + int socket_id, unsigned flags) +{ + int ret; + struct rte_mempool *mp; + + mp = rte_mempool_create_empty(name, n, elt_size, cache_size, + private_data_size, socket_id, flags); + if (mp == NULL) + return NULL; + + /* + * Since we have 4 combinations of the SP/SC/MP/MC examine the flags to + * set the correct index into the table of ops structs. + */ + if ((flags & MEMPOOL_F_SP_PUT) && (flags & MEMPOOL_F_SC_GET)) + ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL); + else if (flags & MEMPOOL_F_SP_PUT) + ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL); + else if (flags & MEMPOOL_F_SC_GET) + ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL); + else + ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL); + + if (ret) + goto fail; + + /* call the mempool priv initializer */ + if (mp_init) + mp_init(mp, mp_init_arg); + + if (rte_mempool_populate_default(mp) < 0) + goto fail; + + /* call the object initializers */ + if (obj_init) + rte_mempool_obj_iter(mp, obj_init, obj_init_arg); + + return mp; + + fail: + rte_mempool_free(mp); + return NULL; +} + +/* + * Create the mempool over already allocated chunk of memory. + * That external memory buffer can consists of physically disjoint pages. + * Setting vaddr to NULL, makes mempool to fallback to rte_mempool_create() + * behavior. + */ +struct rte_mempool * +rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, + unsigned cache_size, unsigned private_data_size, + rte_mempool_ctor_t *mp_init, void *mp_init_arg, + rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, + int socket_id, unsigned flags, void *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift) +{ + struct rte_mempool *mp = NULL; + int ret; + /* no virtual address supplied, use rte_mempool_create() */ + if (vaddr == NULL) + return rte_mempool_create(name, n, elt_size, cache_size, + private_data_size, mp_init, mp_init_arg, + obj_init, obj_init_arg, socket_id, flags); + + /* check that we have both VA and PA */ + if (iova == NULL) { + rte_errno = EINVAL; + return NULL; + } + + /* Check that pg_shift parameter is valid. */ + if (pg_shift > MEMPOOL_PG_SHIFT_MAX) { + rte_errno = EINVAL; + return NULL; + } + + mp = rte_mempool_create_empty(name, n, elt_size, cache_size, + private_data_size, socket_id, flags); + if (mp == NULL) + return NULL; + + /* call the mempool priv initializer */ + if (mp_init) + mp_init(mp, mp_init_arg); + + ret = rte_mempool_populate_iova_tab(mp, vaddr, iova, pg_num, pg_shift, + NULL, NULL); + if (ret < 0 || ret != (int)mp->size) + goto fail; + + /* call the object initializers */ + if (obj_init) + rte_mempool_obj_iter(mp, obj_init, obj_init_arg); + + return mp; + + fail: + rte_mempool_free(mp); return NULL; } /* Return the number of entries in the mempool */ -unsigned -rte_mempool_count(const struct rte_mempool *mp) +unsigned int +rte_mempool_avail_count(const struct rte_mempool *mp) { unsigned count; unsigned lcore_id; - count = rte_ring_count(mp->ring); + count = rte_mempool_ops_get_count(mp); if (mp->cache_size == 0) return count; @@ -728,6 +1018,13 @@ rte_mempool_count(const struct rte_mempool *mp) return count; } +/* return the number of entries allocated from the mempool */ +unsigned int +rte_mempool_in_use_count(const struct rte_mempool *mp) +{ + return mp->size - rte_mempool_avail_count(mp); +} + /* dump the cache status */ static unsigned rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp) @@ -736,7 +1033,7 @@ rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp) unsigned count = 0; unsigned cache_count; - fprintf(f, " cache infos:\n"); + fprintf(f, " internal cache infos:\n"); fprintf(f, " cache_size=%"PRIu32"\n", mp->cache_size); if (mp->cache_size == 0) @@ -744,7 +1041,8 @@ rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp) for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { cache_count = mp->local_cache[lcore_id].len; - fprintf(f, " cache_count[%u]=%u\n", lcore_id, cache_count); + fprintf(f, " cache_count[%u]=%"PRIu32"\n", + lcore_id, cache_count); count += cache_count; } fprintf(f, " total_cache_count=%u\n", count); @@ -770,7 +1068,7 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, /* Force to drop the "const" attribute. This is done only when * DEBUG is enabled */ tmp = (void *) obj_table_const; - obj_table = (void **) tmp; + obj_table = tmp; while (n--) { obj = obj_table[n]; @@ -784,7 +1082,6 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, if (free == 0) { if (cookie != RTE_MEMPOOL_HEADER_COOKIE1) { - rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", obj, (const void *) mp, cookie); @@ -793,7 +1090,6 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2; } else if (free == 1) { if (cookie != RTE_MEMPOOL_HEADER_COOKIE2) { - rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", obj, (const void *) mp, cookie); @@ -803,7 +1099,6 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, } else if (free == 2) { if (cookie != RTE_MEMPOOL_HEADER_COOKIE1 && cookie != RTE_MEMPOOL_HEADER_COOKIE2) { - rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", obj, (const void *) mp, cookie); @@ -813,7 +1108,6 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, tlr = __mempool_get_trailer(obj); cookie = tlr->cookie; if (cookie != RTE_MEMPOOL_TRAILER_COOKIE) { - rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", obj, (const void *) mp, cookie); @@ -867,7 +1161,9 @@ mempool_audit_cache(const struct rte_mempool *mp) return; for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - if (mp->local_cache[lcore_id].len > mp->cache_flushthresh) { + const struct rte_mempool_cache *cache; + cache = &mp->local_cache[lcore_id]; + if (cache->len > cache->flushthresh) { RTE_LOG(CRIT, MEMPOOL, "badness on cache[%u]\n", lcore_id); rte_panic("MEMPOOL: invalid cache len\n"); @@ -904,8 +1200,8 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) fprintf(f, "mempool <%s>@%p\n", mp->name, mp); fprintf(f, " flags=%x\n", mp->flags); - fprintf(f, " ring=<%s>@%p\n", mp->ring->name, mp->ring); - fprintf(f, " phys_addr=0x%" PRIx64 "\n", mp->phys_addr); + fprintf(f, " pool=%p\n", mp->pool_data); + fprintf(f, " iova=0x%" PRIx64 "\n", mp->mz->iova); fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks); fprintf(f, " size=%"PRIu32"\n", mp->size); fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size); @@ -925,7 +1221,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) } cache_count = rte_mempool_dump_cache(f, mp); - common_count = rte_ring_count(mp->ring); + common_count = rte_mempool_ops_get_count(mp); if ((cache_count + common_count) > mp->size) common_count = mp->size - cache_count; fprintf(f, " common_pool_count=%u\n", common_count); @@ -1008,12 +1304,13 @@ void rte_mempool_walk(void (*func)(struct rte_mempool *, void *), { struct rte_tailq_entry *te = NULL; struct rte_mempool_list *mempool_list; + void *tmp_te; mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); - TAILQ_FOREACH(te, mempool_list, next) { + TAILQ_FOREACH_SAFE(te, mempool_list, next, tmp_te) { (*func)((struct rte_mempool *) te->data, arg); }