X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_eal%2Fcommon%2Frte_malloc.c;h=54792ea5b1781e1380578716e26a6f2ca09b9e3d;hb=721ac9f9e063f55570c1c3a2f07cf5a7b5e15aab;hp=23537bc7415db11d1a34378d288a27ea5d96ab38;hpb=15d6dd023c29fc526d16616137844bd91b6afdfe;p=dpdk.git diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c index 23537bc741..54792ea5b1 100644 --- a/lib/librte_eal/common/rte_malloc.c +++ b/lib/librte_eal/common/rte_malloc.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2010-2019 Intel Corporation */ #include @@ -24,6 +24,7 @@ #include #include "malloc_elem.h" #include "malloc_heap.h" +#include "eal_memalloc.h" /* Free the memory space back to heap */ @@ -45,7 +46,13 @@ rte_malloc_socket(const char *type, size_t size, unsigned int align, if (size == 0 || (align && !rte_is_power_of_2(align))) return NULL; - if (!rte_eal_has_hugepages()) + /* if there are no hugepages and if we are not allocating from an + * external heap, use memory from any socket available. checking for + * socket being external may return -1 in case of invalid socket, but + * that's OK - if there are no hugepages, it doesn't matter. + */ + if (rte_malloc_heap_socket_is_external(socket_arg) != 1 && + !rte_eal_has_hugepages()) socket_arg = SOCKET_ID_ANY; return malloc_heap_alloc(type, size, socket_arg, 0, @@ -67,7 +74,18 @@ rte_malloc(const char *type, size_t size, unsigned align) void * rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) { - return rte_malloc_socket(type, size, align, socket); + void *ptr = rte_malloc_socket(type, size, align, socket); + +#ifdef RTE_MALLOC_DEBUG + /* + * If DEBUG is enabled, then freed memory is marked with poison + * value and set to zero on allocation. + * If DEBUG is not enabled then memory is already zeroed. + */ + if (ptr != NULL) + memset(ptr, 0, size); +#endif + return ptr; } /* @@ -98,13 +116,13 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align) } /* - * Resize allocated memory. + * Resize allocated memory on specified heap. */ void * -rte_realloc(void *ptr, size_t size, unsigned align) +rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket) { if (ptr == NULL) - return rte_malloc(NULL, size, align); + return rte_malloc_socket(NULL, size, align, socket); struct malloc_elem *elem = malloc_elem_from_data(ptr); if (elem == NULL) { @@ -113,14 +131,21 @@ rte_realloc(void *ptr, size_t size, unsigned align) } size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); - /* check alignment matches first, and if ok, see if we can resize block */ - if (RTE_PTR_ALIGN(ptr,align) == ptr && + + /* check requested socket id and alignment matches first, and if ok, + * see if we can resize block + */ + if ((socket == SOCKET_ID_ANY || + (unsigned int)socket == elem->heap->socket_id) && + RTE_PTR_ALIGN(ptr, align) == ptr && malloc_heap_resize(elem, size) == 0) return ptr; - /* either alignment is off, or we have no room to expand, - * so move data. */ - void *new_ptr = rte_malloc(NULL, size, align); + /* either requested socket id doesn't match, alignment is off + * or we have no room to expand, + * so move the data. + */ + void *new_ptr = rte_malloc_socket(NULL, size, align, socket); if (new_ptr == NULL) return NULL; const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD; @@ -130,6 +155,15 @@ rte_realloc(void *ptr, size_t size, unsigned align) return new_ptr; } +/* + * Resize allocated memory. + */ +void * +rte_realloc(void *ptr, size_t size, unsigned int align) +{ + return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY); +} + int rte_malloc_validate(const void *ptr, size_t *size) { @@ -149,20 +183,14 @@ rte_malloc_get_socket_stats(int socket, struct rte_malloc_socket_stats *socket_stats) { struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; - int heap_idx, ret = -1; - - rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); + int heap_idx; heap_idx = malloc_socket_to_heap_id(socket); if (heap_idx < 0) - goto unlock; + return -1; - ret = malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], + return malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], socket_stats); -unlock: - rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); - - return ret; } /* @@ -174,14 +202,10 @@ rte_malloc_dump_heaps(FILE *f) struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; unsigned int idx; - rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); - for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { fprintf(f, "Heap id: %u\n", idx); malloc_heap_dump(&mcfg->malloc_heaps[idx], f); } - - rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); } int @@ -255,8 +279,6 @@ rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) unsigned int heap_id; struct rte_malloc_socket_stats sock_stats; - rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); - /* Iterate through all initialised heaps */ for (heap_id = 0; heap_id < RTE_MAX_HEAPS; heap_id++) { struct malloc_heap *heap = &mcfg->malloc_heaps[heap_id]; @@ -273,7 +295,6 @@ rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); } - rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); return; } @@ -327,6 +348,190 @@ find_named_heap(const char *name) return NULL; } +int +rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, + rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + struct malloc_heap *heap = NULL; + struct rte_memseg_list *msl; + unsigned int n; + int ret; + + if (heap_name == NULL || va_addr == NULL || + page_sz == 0 || !rte_is_power_of_2(page_sz) || + RTE_ALIGN(len, page_sz) != len || + !rte_is_aligned(va_addr, page_sz) || + ((len / page_sz) != n_pages && iova_addrs != NULL) || + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == + RTE_HEAP_NAME_MAX_LEN) { + rte_errno = EINVAL; + return -1; + } + rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); + + /* find our heap */ + heap = find_named_heap(heap_name); + if (heap == NULL) { + rte_errno = ENOENT; + ret = -1; + goto unlock; + } + if (heap->socket_id < RTE_MAX_NUMA_NODES) { + /* cannot add memory to internal heaps */ + rte_errno = EPERM; + ret = -1; + goto unlock; + } + n = len / page_sz; + + msl = malloc_heap_create_external_seg(va_addr, iova_addrs, n, page_sz, + heap_name, heap->socket_id); + if (msl == NULL) { + ret = -1; + goto unlock; + } + + rte_spinlock_lock(&heap->lock); + ret = malloc_heap_add_external_memory(heap, msl); + rte_spinlock_unlock(&heap->lock); + +unlock: + rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); + + return ret; +} + +int +rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + struct malloc_heap *heap = NULL; + struct rte_memseg_list *msl; + int ret; + + if (heap_name == NULL || va_addr == NULL || len == 0 || + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == + RTE_HEAP_NAME_MAX_LEN) { + rte_errno = EINVAL; + return -1; + } + rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); + /* find our heap */ + heap = find_named_heap(heap_name); + if (heap == NULL) { + rte_errno = ENOENT; + ret = -1; + goto unlock; + } + if (heap->socket_id < RTE_MAX_NUMA_NODES) { + /* cannot remove memory from internal heaps */ + rte_errno = EPERM; + ret = -1; + goto unlock; + } + + msl = malloc_heap_find_external_seg(va_addr, len); + if (msl == NULL) { + ret = -1; + goto unlock; + } + + rte_spinlock_lock(&heap->lock); + ret = malloc_heap_remove_external_memory(heap, va_addr, len); + rte_spinlock_unlock(&heap->lock); + if (ret != 0) + goto unlock; + + ret = malloc_heap_destroy_external_seg(msl); + +unlock: + rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); + + return ret; +} + +static int +sync_memory(const char *heap_name, void *va_addr, size_t len, bool attach) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + struct malloc_heap *heap = NULL; + struct rte_memseg_list *msl; + int ret; + + if (heap_name == NULL || va_addr == NULL || len == 0 || + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == + RTE_HEAP_NAME_MAX_LEN) { + rte_errno = EINVAL; + return -1; + } + rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); + + /* find our heap */ + heap = find_named_heap(heap_name); + if (heap == NULL) { + rte_errno = ENOENT; + ret = -1; + goto unlock; + } + /* we shouldn't be able to sync to internal heaps */ + if (heap->socket_id < RTE_MAX_NUMA_NODES) { + rte_errno = EPERM; + ret = -1; + goto unlock; + } + + /* find corresponding memseg list to sync to */ + msl = malloc_heap_find_external_seg(va_addr, len); + if (msl == NULL) { + ret = -1; + goto unlock; + } + + if (attach) { + ret = rte_fbarray_attach(&msl->memseg_arr); + if (ret == 0) { + /* notify all subscribers that a new memory area was + * added. + */ + eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, + va_addr, len); + } else { + ret = -1; + goto unlock; + } + } else { + /* notify all subscribers that a memory area is about to + * be removed. + */ + eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, + msl->base_va, msl->len); + ret = rte_fbarray_detach(&msl->memseg_arr); + if (ret < 0) { + ret = -1; + goto unlock; + } + } +unlock: + rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); + return ret; +} + +int +rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len) +{ + return sync_memory(heap_name, va_addr, len, true); +} + +int +rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len) +{ + return sync_memory(heap_name, va_addr, len, false); +} + int rte_malloc_heap_create(const char *heap_name) {