X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_eal%2Fcommon%2Fmalloc_heap.c;h=634ca212f2fe65781c423366d37268d9b63cbb84;hb=a8f8ae1cf9b68f6398b49019ca07a215a57bba41;hp=1973b6e6eca7f5179c45918e0f2af85709f3fe01;hpb=223b7f1d5ef6a22652462a1b703ffcb6db5658cc;p=dpdk.git diff --git a/lib/librte_eal/common/malloc_heap.c b/lib/librte_eal/common/malloc_heap.c index 1973b6e6ec..634ca212f2 100644 --- a/lib/librte_eal/common/malloc_heap.c +++ b/lib/librte_eal/common/malloc_heap.c @@ -20,11 +20,13 @@ #include #include #include +#include #include #include #include "eal_internal_cfg.h" #include "eal_memalloc.h" +#include "eal_memcfg.h" #include "malloc_elem.h" #include "malloc_heap.h" #include "malloc_mp.h" @@ -94,7 +96,7 @@ malloc_heap_add_memory(struct malloc_heap *heap, struct rte_memseg_list *msl, { struct malloc_elem *elem = start; - malloc_elem_init(elem, heap, msl, len); + malloc_elem_init(elem, heap, msl, len, elem, len); malloc_elem_insert(elem); @@ -294,7 +296,6 @@ alloc_pages_on_heap(struct malloc_heap *heap, uint64_t pg_sz, size_t elt_size, size_t alloc_sz; int allocd_pages; void *ret, *map_addr; - uint64_t mask; alloc_sz = (size_t)pg_sz * n_segs; @@ -322,14 +323,44 @@ alloc_pages_on_heap(struct malloc_heap *heap, uint64_t pg_sz, size_t elt_size, goto fail; } - if (mcfg->dma_maskbits) { - mask = ~((1ULL << mcfg->dma_maskbits) - 1); - if (rte_eal_check_dma_mask(mask)) { + /* + * Once we have all the memseg lists configured, if there is a dma mask + * set, check iova addresses are not out of range. Otherwise the device + * setting the dma mask could have problems with the mapped memory. + * + * There are two situations when this can happen: + * 1) memory initialization + * 2) dynamic memory allocation + * + * For 1), an error when checking dma mask implies app can not be + * executed. For 2) implies the new memory can not be added. + */ + if (mcfg->dma_maskbits && + rte_mem_check_dma_mask_thread_unsafe(mcfg->dma_maskbits)) { + /* + * Currently this can only happen if IOMMU is enabled + * and the address width supported by the IOMMU hw is + * not enough for using the memory mapped IOVAs. + * + * If IOVA is VA, advice to try with '--iova-mode pa' + * which could solve some situations when IOVA VA is not + * really needed. + */ + RTE_LOG(ERR, EAL, + "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask\n", + __func__); + + /* + * If IOVA is VA and it is possible to run with IOVA PA, + * because user is root, give and advice for solving the + * problem. + */ + if ((rte_eal_iova_mode() == RTE_IOVA_VA) && + rte_eal_using_phys_addrs()) RTE_LOG(ERR, EAL, - "%s(): couldn't allocate memory due to DMA mask\n", + "%s(): Please try initializing EAL with --iova-mode=pa parameter\n", __func__); - goto fail; - } + goto fail; } /* add newly minted memsegs to malloc heap */ @@ -456,10 +487,9 @@ try_expand_heap(struct malloc_heap *heap, uint64_t pg_sz, size_t elt_size, int socket, unsigned int flags, size_t align, size_t bound, bool contig) { - struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; int ret; - rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); + rte_mcfg_mem_write_lock(); if (rte_eal_process_type() == RTE_PROC_PRIMARY) { ret = try_expand_heap_primary(heap, pg_sz, elt_size, socket, @@ -469,7 +499,7 @@ try_expand_heap(struct malloc_heap *heap, uint64_t pg_sz, size_t elt_size, flags, align, bound, contig); } - rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); + rte_mcfg_mem_write_unlock(); return ret; } @@ -792,7 +822,6 @@ malloc_heap_free_pages(void *aligned_start, size_t aligned_len) int malloc_heap_free(struct malloc_elem *elem) { - struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; struct malloc_heap *heap; void *start, *aligned_start, *end, *aligned_end; size_t len, aligned_len, page_sz; @@ -828,6 +857,13 @@ malloc_heap_free(struct malloc_elem *elem) if (elem->size < page_sz) goto free_unlock; + /* if user requested to match allocations, the sizes must match - if not, + * we will defer freeing these hugepages until the entire original allocation + * can be freed + */ + if (internal_config.match_allocations && elem->size != elem->orig_size) + goto free_unlock; + /* probably, but let's make sure, as we may not be using up full page */ start = elem; len = elem->size; @@ -899,7 +935,7 @@ malloc_heap_free(struct malloc_elem *elem) /* now we can finally free us some pages */ - rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); + rte_mcfg_mem_write_lock(); /* * we allow secondary processes to clear the heap of this allocated @@ -954,7 +990,7 @@ malloc_heap_free(struct malloc_elem *elem) RTE_LOG(DEBUG, EAL, "Heap on socket %d was shrunk by %zdMB\n", msl->socket_id, aligned_len >> 20ULL); - rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); + rte_mcfg_mem_write_unlock(); free_unlock: rte_spinlock_unlock(&(heap->lock)); return ret; @@ -1038,12 +1074,9 @@ malloc_heap_dump(struct malloc_heap *heap, FILE *f) } static int -destroy_seg(struct malloc_elem *elem, size_t len) +destroy_elem(struct malloc_elem *elem, size_t len) { struct malloc_heap *heap = elem->heap; - struct rte_memseg_list *msl; - - msl = elem->msl; /* notify all subscribers that a memory area is going to be removed */ eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, elem, len); @@ -1056,19 +1089,13 @@ destroy_seg(struct malloc_elem *elem, size_t len) memset(elem, 0, sizeof(*elem)); - /* destroy the fbarray backing this memory */ - if (rte_fbarray_destroy(&msl->memseg_arr) < 0) - return -1; - - /* reset the memseg list */ - memset(msl, 0, sizeof(*msl)); - return 0; } -int -malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr, - rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) +struct rte_memseg_list * +malloc_heap_create_external_seg(void *va_addr, rte_iova_t iova_addrs[], + unsigned int n_pages, size_t page_sz, const char *seg_name, + unsigned int socket_id) { struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; char fbarray_name[RTE_FBARRAY_NAME_LEN]; @@ -1088,17 +1115,17 @@ malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr, if (msl == NULL) { RTE_LOG(ERR, EAL, "Couldn't find empty memseg list\n"); rte_errno = ENOSPC; - return -1; + return NULL; } - snprintf(fbarray_name, sizeof(fbarray_name) - 1, "%s_%p", - heap->name, va_addr); + snprintf(fbarray_name, sizeof(fbarray_name), "%s_%p", + seg_name, va_addr); /* create the backing fbarray */ if (rte_fbarray_init(&msl->memseg_arr, fbarray_name, n_pages, sizeof(struct rte_memseg)) < 0) { RTE_LOG(ERR, EAL, "Couldn't create fbarray backing the memseg list\n"); - return -1; + return NULL; } arr = &msl->memseg_arr; @@ -1114,32 +1141,95 @@ malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr, ms->len = page_sz; ms->nchannel = rte_memory_get_nchannel(); ms->nrank = rte_memory_get_nrank(); - ms->socket_id = heap->socket_id; + ms->socket_id = socket_id; } /* set up the memseg list */ msl->base_va = va_addr; msl->page_sz = page_sz; - msl->socket_id = heap->socket_id; + msl->socket_id = socket_id; msl->len = seg_len; msl->version = 0; msl->external = 1; + return msl; +} + +struct extseg_walk_arg { + void *va_addr; + size_t len; + struct rte_memseg_list *msl; +}; + +static int +extseg_walk(const struct rte_memseg_list *msl, void *arg) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + struct extseg_walk_arg *wa = arg; + + if (msl->base_va == wa->va_addr && msl->len == wa->len) { + unsigned int found_idx; + + /* msl is const */ + found_idx = msl - mcfg->memsegs; + wa->msl = &mcfg->memsegs[found_idx]; + return 1; + } + return 0; +} + +struct rte_memseg_list * +malloc_heap_find_external_seg(void *va_addr, size_t len) +{ + struct extseg_walk_arg wa; + int res; + + wa.va_addr = va_addr; + wa.len = len; + + res = rte_memseg_list_walk_thread_unsafe(extseg_walk, &wa); + + if (res != 1) { + /* 0 means nothing was found, -1 shouldn't happen */ + if (res == 0) + rte_errno = ENOENT; + return NULL; + } + return wa.msl; +} + +int +malloc_heap_destroy_external_seg(struct rte_memseg_list *msl) +{ + /* destroy the fbarray backing this memory */ + if (rte_fbarray_destroy(&msl->memseg_arr) < 0) + return -1; + + /* reset the memseg list */ + memset(msl, 0, sizeof(*msl)); + + return 0; +} + +int +malloc_heap_add_external_memory(struct malloc_heap *heap, + struct rte_memseg_list *msl) +{ /* erase contents of new memory */ - memset(va_addr, 0, seg_len); + memset(msl->base_va, 0, msl->len); /* now, add newly minted memory to the malloc heap */ - malloc_heap_add_memory(heap, msl, va_addr, seg_len); + malloc_heap_add_memory(heap, msl, msl->base_va, msl->len); - heap->total_size += seg_len; + heap->total_size += msl->len; /* all done! */ RTE_LOG(DEBUG, EAL, "Added segment for heap %s starting at %p\n", - heap->name, va_addr); + heap->name, msl->base_va); /* notify all subscribers that a new memory area has been added */ eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, - va_addr, seg_len); + msl->base_va, msl->len); return 0; } @@ -1169,7 +1259,7 @@ malloc_heap_remove_external_memory(struct malloc_heap *heap, void *va_addr, rte_errno = EBUSY; return -1; } - return destroy_seg(elem, len); + return destroy_elem(elem, len); } int @@ -1230,6 +1320,10 @@ rte_eal_malloc_heap_init(void) struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; unsigned int i; + if (internal_config.match_allocations) { + RTE_LOG(DEBUG, EAL, "Hugepages will be freed exactly as allocated.\n"); + } + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { /* assign min socket ID to external heaps */ mcfg->next_socket_id = EXTERNAL_HEAP_MIN_SOCKET_ID; @@ -1240,7 +1334,7 @@ rte_eal_malloc_heap_init(void) char heap_name[RTE_HEAP_NAME_MAX_LEN]; int socket_id = rte_socket_id_by_idx(i); - snprintf(heap_name, sizeof(heap_name) - 1, + snprintf(heap_name, sizeof(heap_name), "socket_%i", socket_id); strlcpy(heap->name, heap_name, RTE_HEAP_NAME_MAX_LEN); heap->socket_id = socket_id; @@ -1250,7 +1344,7 @@ rte_eal_malloc_heap_init(void) if (register_mp_requests()) { RTE_LOG(ERR, EAL, "Couldn't register malloc multiprocess actions\n"); - rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); + rte_mcfg_mem_read_unlock(); return -1; } @@ -1258,7 +1352,7 @@ rte_eal_malloc_heap_init(void) * even come before primary itself is fully initialized, and secondaries * do not need to initialize the heap. */ - rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); + rte_mcfg_mem_read_unlock(); /* secondary process does not need to initialize anything */ if (rte_eal_process_type() != RTE_PROC_PRIMARY)