From a7cece2ead257183daed2e9a2b97cb82033daa9d Mon Sep 17 00:00:00 2001 From: Tomasz Jozwiak Date: Fri, 1 Mar 2019 09:46:16 +0100 Subject: [PATCH] malloc: add NUMA-aware realloc function Currently, rte_realloc will not respect original allocation's NUMA node when memory cannot be resized, and there is no NUMA-aware equivalent of rte_realloc. This patch adds such a function. The new API will ensure that reallocated memory stays on requested NUMA node, as well as allow moving allocated memory to a different NUMA node. Signed-off-by: Tomasz Jozwiak Acked-by: Anatoly Burakov --- app/test/test_malloc.c | 53 +++++++++++++++++++++- lib/librte_eal/common/include/rte_malloc.h | 29 +++++++++++- lib/librte_eal/common/rte_malloc.c | 34 ++++++++++---- lib/librte_eal/rte_eal_version.map | 1 + 4 files changed, 104 insertions(+), 13 deletions(-) diff --git a/app/test/test_malloc.c b/app/test/test_malloc.c index 6b6c6fec11..f0e608cdaf 100644 --- a/app/test/test_malloc.c +++ b/app/test/test_malloc.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2010-2019 Intel Corporation */ #include @@ -26,6 +26,13 @@ #define N 10000 + +static int +is_mem_on_socket(int32_t socket); + +static int32_t +addr_to_socket(void *addr); + /* * Malloc * ====== @@ -542,7 +549,49 @@ test_realloc(void) return -1; } rte_free(ptr12); - return 0; + + /* check realloc_socket part */ + int32_t socket_count = 0, socket_allocated, socket; + int ret = -1; + size_t size = 1024; + + ptr1 = NULL; + for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) { + if (is_mem_on_socket(socket)) { + int j = 2; + + socket_count++; + while (j--) { + /* j == 1 -> resizing */ + ptr2 = rte_realloc_socket(ptr1, size, + RTE_CACHE_LINE_SIZE, + socket); + if (ptr2 == NULL) { + printf("NULL pointer returned from rte_realloc_socket\n"); + goto end; + } + + ptr1 = ptr2; + socket_allocated = addr_to_socket(ptr2); + if (socket_allocated != socket) { + printf("Requested socket (%d) doesn't mach allocated one (%d)\n", + socket, socket_allocated); + goto end; + } + size += RTE_CACHE_LINE_SIZE; + } + } + } + + /* Print warnign if only a single socket, but don't fail the test */ + if (socket_count < 2) + printf("WARNING: realloc_socket test needs memory on multiple sockets!\n"); + + ret = 0; +end: + rte_free(ptr1); + + return ret; } static int diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h index 54a12467a4..5fbde4ed8b 100644 --- a/lib/librte_eal/common/include/rte_malloc.h +++ b/lib/librte_eal/common/include/rte_malloc.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2010-2019 Intel Corporation */ #ifndef _RTE_MALLOC_H_ @@ -129,7 +129,32 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align); * - Otherwise, the pointer to the reallocated memory. */ void * -rte_realloc(void *ptr, size_t size, unsigned align); +rte_realloc(void *ptr, size_t size, unsigned int align); + +/** + * Replacement function for realloc(), using huge-page memory. Reserved area + * memory is resized, preserving contents. In NUMA systems, the new area + * resides on requested NUMA socket. + * + * @param ptr + * Pointer to already allocated memory + * @param size + * Size (in bytes) of new area. If this is 0, memory is freed. + * @param align + * If 0, the return is a pointer that is suitably aligned for any kind of + * variable (in the same manner as malloc()). + * Otherwise, the return is a pointer that is a multiple of *align*. In + * this case, it must obviously be a power of two. (Minimum alignment is the + * cacheline size, i.e. 64-bytes) + * @param socket + * NUMA socket to allocate memory on. + * @return + * - NULL on error. Not enough memory, or invalid arguments (size is 0, + * align is not a power of two). + * - Otherwise, the pointer to the reallocated memory. + */ +void * __rte_experimental +rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket); /** * This function allocates memory from the huge-page area of memory. The memory diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c index b39de3c99e..eebd0674b3 100644 --- a/lib/librte_eal/common/rte_malloc.c +++ b/lib/librte_eal/common/rte_malloc.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2010-2019 Intel Corporation */ #include @@ -105,13 +105,13 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align) } /* - * Resize allocated memory. + * Resize allocated memory on specified heap. */ void * -rte_realloc(void *ptr, size_t size, unsigned align) +rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket) { if (ptr == NULL) - return rte_malloc(NULL, size, align); + return rte_malloc_socket(NULL, size, align, socket); struct malloc_elem *elem = malloc_elem_from_data(ptr); if (elem == NULL) { @@ -120,14 +120,21 @@ rte_realloc(void *ptr, size_t size, unsigned align) } size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); - /* check alignment matches first, and if ok, see if we can resize block */ - if (RTE_PTR_ALIGN(ptr,align) == ptr && + + /* check requested socket id and alignment matches first, and if ok, + * see if we can resize block + */ + if ((socket == SOCKET_ID_ANY || + (unsigned int)socket == elem->heap->socket_id) && + RTE_PTR_ALIGN(ptr, align) == ptr && malloc_heap_resize(elem, size) == 0) return ptr; - /* either alignment is off, or we have no room to expand, - * so move data. */ - void *new_ptr = rte_malloc(NULL, size, align); + /* either requested socket id doesn't match, alignment is off + * or we have no room to expand, + * so move the data. + */ + void *new_ptr = rte_malloc_socket(NULL, size, align, socket); if (new_ptr == NULL) return NULL; const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD; @@ -137,6 +144,15 @@ rte_realloc(void *ptr, size_t size, unsigned align) return new_ptr; } +/* + * Resize allocated memory. + */ +void * +rte_realloc(void *ptr, size_t size, unsigned int align) +{ + return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY); +} + int rte_malloc_validate(const void *ptr, size_t *size) { diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map index eb5f7b9cbd..935e918a75 100644 --- a/lib/librte_eal/rte_eal_version.map +++ b/lib/librte_eal/rte_eal_version.map @@ -359,6 +359,7 @@ EXPERIMENTAL { rte_mp_request_async; rte_mp_sendmsg; rte_option_register; + rte_realloc_socket; rte_service_lcore_attr_get; rte_service_lcore_attr_reset_all; rte_service_may_be_active; -- 2.20.1