X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2Flibrte_mempool%2Frte_mempool.c;h=b9f3fbd614b5f4bd8dcbcaba2a65cd2b7faebf90;hb=3cd73a1a1c4d99439a6883c37a4ba4b561325615;hp=7260ce0beb55dae21ca9cf7409ec7d048f944913;hpb=028669bc9f0d1b44a7656920733ab98d626aa373;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index 7260ce0beb..b9f3fbd614 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -31,8 +30,10 @@ #include #include #include +#include #include "rte_mempool.h" +#include "rte_mempool_trace.h" TAILQ_HEAD(rte_mempool_list, rte_tailq_entry); @@ -45,6 +46,7 @@ EAL_REGISTER_TAILQ(rte_mempool_tailq) #define CALC_CACHE_FLUSHTHRESH(c) \ ((typeof(c))((c) * CACHE_FLUSHTHRESH_MULTIPLIER)) +#if defined(RTE_ARCH_X86) /* * return the greatest common divisor between a and b (fast algorithm) * @@ -74,12 +76,13 @@ static unsigned get_gcd(unsigned a, unsigned b) } /* - * Depending on memory configuration, objects addresses are spread + * Depending on memory configuration on x86 arch, objects addresses are spread * between channels and ranks in RAM: the pool allocator will add * padding between objects. This function return the new size of the * object. */ -static unsigned optimize_object_size(unsigned obj_size) +static unsigned int +arch_mem_object_align(unsigned int obj_size) { unsigned nrank, nchan; unsigned new_obj_size; @@ -99,6 +102,13 @@ static unsigned optimize_object_size(unsigned obj_size) new_obj_size++; return new_obj_size * RTE_MEMPOOL_ALIGN; } +#else +static unsigned int +arch_mem_object_align(unsigned int obj_size) +{ + return obj_size; +} +#endif struct pagesz_walk_arg { int socket_id; @@ -137,7 +147,7 @@ get_min_page_size(int socket_id) rte_memseg_list_walk(find_min_pagesz, &wa); - return wa.min == SIZE_MAX ? (size_t) getpagesize() : wa.min; + return wa.min == SIZE_MAX ? (size_t) rte_mem_page_size() : wa.min; } @@ -234,8 +244,8 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, */ if ((flags & MEMPOOL_F_NO_SPREAD) == 0) { unsigned new_size; - new_size = optimize_object_size(sz->header_size + sz->elt_size + - sz->trailer_size); + new_size = arch_mem_object_align + (sz->header_size + sz->elt_size + sz->trailer_size); sz->trailer_size = new_size - sz->header_size - sz->elt_size; } @@ -329,10 +339,10 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr; else - off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_CACHE_LINE_SIZE) - vaddr; + off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_MEMPOOL_ALIGN) - vaddr; if (off > len) { - ret = -EINVAL; + ret = 0; goto fail; } @@ -343,12 +353,14 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, /* not enough room to store one object */ if (i == 0) { - ret = -EINVAL; + ret = 0; goto fail; } STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next); mp->nb_mem_chunks++; + + rte_mempool_trace_populate_iova(mp, vaddr, iova, len, free_cb, opaque); return i; fail: @@ -356,6 +368,19 @@ fail: return ret; } +static rte_iova_t +get_iova(void *addr) +{ + struct rte_memseg *ms; + + /* try registered memory first */ + ms = rte_mem_virt2memseg(addr, NULL); + if (ms == NULL || ms->iova == RTE_BAD_IOVA) + /* fall back to actual physical address */ + return rte_mem_virt2iova(addr); + return ms->iova + RTE_PTR_DIFF(addr, ms->addr); +} + /* Populate the mempool with a virtual area. Return the number of * objects added, or a negative value on error. */ @@ -368,38 +393,35 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t off, phys_len; int ret, cnt = 0; - /* address and len must be page-aligned */ - if (RTE_PTR_ALIGN_CEIL(addr, pg_sz) != addr) - return -EINVAL; - if (RTE_ALIGN_CEIL(len, pg_sz) != len) - return -EINVAL; - if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG) return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, len, free_cb, opaque); - for (off = 0; off + pg_sz <= len && + for (off = 0; off < len && mp->populated_size < mp->size; off += phys_len) { - iova = rte_mem_virt2iova(addr + off); - - if (iova == RTE_BAD_IOVA && rte_eal_has_hugepages()) { - ret = -EINVAL; - goto fail; - } + iova = get_iova(addr + off); /* populate with the largest group of contiguous pages */ - for (phys_len = pg_sz; off + phys_len < len; phys_len += pg_sz) { + for (phys_len = RTE_MIN( + (size_t)(RTE_PTR_ALIGN_CEIL(addr + off + 1, pg_sz) - + (addr + off)), + len - off); + off + phys_len < len; + phys_len = RTE_MIN(phys_len + pg_sz, len - off)) { rte_iova_t iova_tmp; - iova_tmp = rte_mem_virt2iova(addr + off + phys_len); + iova_tmp = get_iova(addr + off + phys_len); - if (iova_tmp != iova + phys_len) + if (iova_tmp == RTE_BAD_IOVA || + iova_tmp != iova + phys_len) break; } ret = rte_mempool_populate_iova(mp, addr + off, iova, phys_len, free_cb, opaque); + if (ret == 0) + continue; if (ret < 0) goto fail; /* no need to call the free callback for next chunks */ @@ -407,6 +429,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, cnt += ret; } + rte_mempool_trace_populate_virt(mp, addr, len, pg_sz, free_cb, opaque); return cnt; fail: @@ -414,6 +437,32 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, return ret; } +/* Get the minimal page size used in a mempool before populating it. */ +int +rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz) +{ + bool need_iova_contig_obj; + bool alloc_in_ext_mem; + int ret; + + /* check if we can retrieve a valid socket ID */ + ret = rte_malloc_heap_socket_is_external(mp->socket_id); + if (ret < 0) + return -EINVAL; + alloc_in_ext_mem = (ret == 1); + need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG); + + if (!need_iova_contig_obj) + *pg_sz = 0; + else if (rte_eal_has_hugepages() || alloc_in_ext_mem) + *pg_sz = get_min_page_size(mp->socket_id); + else + *pg_sz = rte_mem_page_size(); + + rte_mempool_trace_get_page_size(mp, *pg_sz); + return 0; +} + /* Default function to populate the mempool: allocate memory in memzones, * and populate them. Return the number of objects added, or a negative * value on error. @@ -425,28 +474,21 @@ rte_mempool_populate_default(struct rte_mempool *mp) char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; ssize_t mem_size; - size_t align, pg_sz, pg_shift; + size_t align, pg_sz, pg_shift = 0; rte_iova_t iova; unsigned mz_id, n; int ret; - bool no_contig, try_contig, no_pageshift, external; + bool need_iova_contig_obj; + size_t max_alloc_size = SIZE_MAX; ret = mempool_ops_alloc_once(mp); if (ret != 0) return ret; - /* check if we can retrieve a valid socket ID */ - ret = rte_malloc_heap_socket_is_external(mp->socket_id); - if (ret < 0) - return -EINVAL; - external = ret; - /* mempool must not be populated */ if (mp->nb_mem_chunks != 0) return -EEXIST; - no_contig = mp->flags & MEMPOOL_F_NO_IOVA_CONTIG; - /* * the following section calculates page shift and page size values. * @@ -462,17 +504,15 @@ rte_mempool_populate_default(struct rte_mempool *mp) * then just set page shift and page size to 0, because the user has * indicated that there's no need to care about anything. * - * if we do need contiguous objects, there is also an option to reserve - * the entire mempool memory as one contiguous block of memory, in - * which case the page shift and alignment wouldn't matter as well. + * if we do need contiguous objects (if a mempool driver has its + * own calc_size() method returning min_chunk_size = mem_size), + * there is also an option to reserve the entire mempool memory + * as one contiguous block of memory. * * if we require contiguous objects, but not necessarily the entire - * mempool reserved space to be contiguous, then there are two options. - * - * if our IO addresses are virtual, not actual physical (IOVA as VA - * case), then no page shift needed - our memory allocation will give us - * contiguous IO memory as far as the hardware is concerned, so - * act as if we're getting contiguous memory. + * mempool reserved space to be contiguous, pg_sz will be != 0, + * and the default ops->populate() will take care of not placing + * objects across pages. * * if our IO addresses are physical, we may get memory from bigger * pages, or we might get memory from smaller pages, and how much of it @@ -483,46 +523,23 @@ rte_mempool_populate_default(struct rte_mempool *mp) * wasting some space this way, but it's much nicer than looping around * trying to reserve each and every page size. * - * However, since size calculation will produce page-aligned sizes, it - * makes sense to first try and see if we can reserve the entire memzone - * in one contiguous chunk as well (otherwise we might end up wasting a - * 1G page on a 10MB memzone). If we fail to get enough contiguous - * memory, then we'll go and reserve space page-by-page. - * - * We also have to take into account the fact that memory that we're - * going to allocate from can belong to an externally allocated memory - * area, in which case the assumption of IOVA as VA mode being - * synonymous with IOVA contiguousness will not hold. We should also try - * to go for contiguous memory even if we're in no-huge mode, because - * external memory may in fact be IOVA-contiguous. + * If we fail to get enough contiguous memory, then we'll go and + * reserve space in smaller chunks. */ - external = rte_malloc_heap_socket_is_external(mp->socket_id) == 1; - no_pageshift = no_contig || - (!external && rte_eal_iova_mode() == RTE_IOVA_VA); - try_contig = !no_contig && !no_pageshift && - (rte_eal_has_hugepages() || external); - - if (no_pageshift) { - pg_sz = 0; - pg_shift = 0; - } else if (try_contig) { - pg_sz = get_min_page_size(mp->socket_id); - pg_shift = rte_bsf32(pg_sz); - } else { - pg_sz = getpagesize(); + + need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG); + ret = rte_mempool_get_page_size(mp, &pg_sz); + if (ret < 0) + return ret; + + if (pg_sz != 0) pg_shift = rte_bsf32(pg_sz); - } for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { size_t min_chunk_size; - unsigned int flags; - if (try_contig || no_pageshift) - mem_size = rte_mempool_ops_calc_mem_size(mp, n, - 0, &min_chunk_size, &align); - else - mem_size = rte_mempool_ops_calc_mem_size(mp, n, - pg_shift, &min_chunk_size, &align); + mem_size = rte_mempool_ops_calc_mem_size( + mp, n, pg_shift, &min_chunk_size, &align); if (mem_size < 0) { ret = mem_size; @@ -536,78 +553,54 @@ rte_mempool_populate_default(struct rte_mempool *mp) goto fail; } - flags = mz_flags; - /* if we're trying to reserve contiguous memory, add appropriate * memzone flag. */ - if (try_contig) - flags |= RTE_MEMZONE_IOVA_CONTIG; + if (min_chunk_size == (size_t)mem_size) + mz_flags |= RTE_MEMZONE_IOVA_CONTIG; - mz = rte_memzone_reserve_aligned(mz_name, mem_size, - mp->socket_id, flags, align); + /* Allocate a memzone, retrying with a smaller area on ENOMEM */ + do { + mz = rte_memzone_reserve_aligned(mz_name, + RTE_MIN((size_t)mem_size, max_alloc_size), + mp->socket_id, mz_flags, align); - /* if we were trying to allocate contiguous memory, failed and - * minimum required contiguous chunk fits minimum page, adjust - * memzone size to the page size, and try again. - */ - if (mz == NULL && try_contig && min_chunk_size <= pg_sz) { - try_contig = false; - flags &= ~RTE_MEMZONE_IOVA_CONTIG; - - mem_size = rte_mempool_ops_calc_mem_size(mp, n, - pg_shift, &min_chunk_size, &align); - if (mem_size < 0) { - ret = mem_size; - goto fail; - } + if (mz != NULL || rte_errno != ENOMEM) + break; + + max_alloc_size = RTE_MIN(max_alloc_size, + (size_t)mem_size) / 2; + } while (mz == NULL && max_alloc_size >= min_chunk_size); - mz = rte_memzone_reserve_aligned(mz_name, mem_size, - mp->socket_id, flags, align); - } - /* don't try reserving with 0 size if we were asked to reserve - * IOVA-contiguous memory. - */ - if (min_chunk_size < (size_t)mem_size && mz == NULL) { - /* not enough memory, retry with the biggest zone we - * have - */ - mz = rte_memzone_reserve_aligned(mz_name, 0, - mp->socket_id, flags, - RTE_MAX(pg_sz, align)); - } if (mz == NULL) { ret = -rte_errno; goto fail; } - if (mz->len < min_chunk_size) { - rte_memzone_free(mz); - ret = -ENOMEM; - goto fail; - } - - if (no_contig) - iova = RTE_BAD_IOVA; - else + if (need_iova_contig_obj) iova = mz->iova; + else + iova = RTE_BAD_IOVA; - if (no_pageshift || try_contig) + if (pg_sz == 0 || (mz_flags & RTE_MEMZONE_IOVA_CONTIG)) ret = rte_mempool_populate_iova(mp, mz->addr, iova, mz->len, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); else ret = rte_mempool_populate_virt(mp, mz->addr, - RTE_ALIGN_FLOOR(mz->len, pg_sz), pg_sz, + mz->len, pg_sz, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); + if (ret == 0) /* should not happen */ + ret = -ENOBUFS; if (ret < 0) { rte_memzone_free(mz); goto fail; } } + rte_mempool_trace_populate_default(mp); return mp->size; fail: @@ -624,7 +617,7 @@ get_anon_size(const struct rte_mempool *mp) size_t min_chunk_size; size_t align; - pg_sz = getpagesize(); + pg_sz = rte_mem_page_size(); pg_shift = rte_bsf32(pg_sz); size = rte_mempool_ops_calc_mem_size(mp, mp->size, pg_shift, &min_chunk_size, &align); @@ -648,7 +641,7 @@ rte_mempool_memchunk_anon_free(struct rte_mempool_memhdr *memhdr, if (size < 0) return; - munmap(opaque, size); + rte_mem_unmap(opaque, size); } /* populate the mempool with an anonymous mapping */ @@ -666,8 +659,10 @@ rte_mempool_populate_anon(struct rte_mempool *mp) } ret = mempool_ops_alloc_once(mp); - if (ret != 0) - return ret; + if (ret < 0) { + rte_errno = -ret; + return 0; + } size = get_anon_size(mp); if (size < 0) { @@ -676,24 +671,26 @@ rte_mempool_populate_anon(struct rte_mempool *mp) } /* get chunk of virtually continuous memory */ - addr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) { - rte_errno = errno; + addr = rte_mem_map(NULL, size, RTE_PROT_READ | RTE_PROT_WRITE, + RTE_MAP_SHARED | RTE_MAP_ANONYMOUS, -1, 0); + if (addr == NULL) return 0; - } /* can't use MMAP_LOCKED, it does not exist on BSD */ - if (mlock(addr, size) < 0) { - rte_errno = errno; - munmap(addr, size); + if (rte_mem_lock(addr, size) < 0) { + rte_mem_unmap(addr, size); return 0; } - ret = rte_mempool_populate_virt(mp, addr, size, getpagesize(), + ret = rte_mempool_populate_virt(mp, addr, size, rte_mem_page_size(), rte_mempool_memchunk_anon_free, addr); - if (ret == 0) + if (ret == 0) /* should not happen */ + ret = -ENOBUFS; + if (ret < 0) { + rte_errno = -ret; goto fail; + } + rte_mempool_trace_populate_anon(mp); return mp->populated_size; fail: @@ -725,6 +722,7 @@ rte_mempool_free(struct rte_mempool *mp) } rte_mcfg_tailq_write_unlock(); + rte_mempool_trace_free(mp); rte_mempool_free_memchunks(mp); rte_mempool_ops_free(mp); rte_memzone_free(mp->mz); @@ -763,6 +761,7 @@ rte_mempool_cache_create(uint32_t size, int socket_id) mempool_cache_init(cache, size); + rte_mempool_trace_cache_create(size, socket_id, cache); return cache; } @@ -774,6 +773,7 @@ rte_mempool_cache_create(uint32_t size, int socket_id) void rte_mempool_cache_free(struct rte_mempool_cache *cache) { + rte_mempool_trace_cache_free(cache); rte_free(cache); } @@ -904,6 +904,8 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, rte_mcfg_tailq_write_unlock(); rte_mcfg_mempool_write_unlock(); + rte_mempool_trace_create_empty(name, n, elt_size, cache_size, + private_data_size, flags, mp); return mp; exit_unlock: @@ -956,6 +958,9 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, if (obj_init) rte_mempool_obj_iter(mp, obj_init, obj_init_arg); + rte_mempool_trace_create(name, n, elt_size, cache_size, + private_data_size, mp_init, mp_init_arg, obj_init, + obj_init_arg, flags, mp); return mp; fail: @@ -1191,6 +1196,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) unsigned lcore_id; #endif struct rte_mempool_memhdr *memhdr; + struct rte_mempool_ops *ops; unsigned common_count; unsigned cache_count; size_t mem_len = 0; @@ -1200,6 +1206,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) fprintf(f, "mempool <%s>@%p\n", mp->name, mp); fprintf(f, " flags=%x\n", mp->flags); + fprintf(f, " socket_id=%d\n", mp->socket_id); fprintf(f, " pool=%p\n", mp->pool_data); fprintf(f, " iova=0x%" PRIx64 "\n", mp->mz->iova); fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks); @@ -1213,6 +1220,10 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) fprintf(f, " private_data_size=%"PRIu32"\n", mp->private_data_size); + fprintf(f, " ops_index=%d\n", mp->ops_index); + ops = rte_mempool_get_ops(mp->ops_index); + fprintf(f, " ops_name: <%s>\n", (ops != NULL) ? ops->name : "NA"); + STAILQ_FOREACH(memhdr, &mp->mem_list, next) mem_len += memhdr->len; if (mem_len != 0) {