From dec5e73d78e011af5c1bd4e18ae406df505db13b Mon Sep 17 00:00:00 2001 From: Intel Date: Thu, 20 Dec 2012 00:00:00 +0100 Subject: [PATCH] memory: add numa-awareness to malloc Signed-off-by: Intel --- config/defconfig_i686-default-linuxapp-gcc | 1 - config/defconfig_i686-default-linuxapp-icc | 1 - config/defconfig_x86_64-default-linuxapp-gcc | 1 - config/defconfig_x86_64-default-linuxapp-icc | 1 - lib/librte_eal/common/Makefile | 2 +- .../common/include/rte_malloc_heap.h | 56 +++++++++++ lib/librte_malloc/malloc_heap.h | 31 +++--- lib/librte_malloc/rte_malloc.c | 48 ++++++++-- lib/librte_malloc/rte_malloc.h | 96 ++++++++++++++++++- 9 files changed, 203 insertions(+), 34 deletions(-) create mode 100644 lib/librte_eal/common/include/rte_malloc_heap.h diff --git a/config/defconfig_i686-default-linuxapp-gcc b/config/defconfig_i686-default-linuxapp-gcc index cfe8aa0469..4033194d02 100644 --- a/config/defconfig_i686-default-linuxapp-gcc +++ b/config/defconfig_i686-default-linuxapp-gcc @@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n CONFIG_RTE_LIBRTE_MALLOC=y CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M -CONFIG_RTE_MALLOC_PER_NUMA_NODE=y # # Compile librte_cmdline diff --git a/config/defconfig_i686-default-linuxapp-icc b/config/defconfig_i686-default-linuxapp-icc index 9057084331..2850b6e815 100644 --- a/config/defconfig_i686-default-linuxapp-icc +++ b/config/defconfig_i686-default-linuxapp-icc @@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n CONFIG_RTE_LIBRTE_MALLOC=y CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M -CONFIG_RTE_MALLOC_PER_NUMA_NODE=y # # Compile librte_cmdline diff --git a/config/defconfig_x86_64-default-linuxapp-gcc b/config/defconfig_x86_64-default-linuxapp-gcc index 0df200824f..5fc5ad0478 100644 --- a/config/defconfig_x86_64-default-linuxapp-gcc +++ b/config/defconfig_x86_64-default-linuxapp-gcc @@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n CONFIG_RTE_LIBRTE_MALLOC=y CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M -CONFIG_RTE_MALLOC_PER_NUMA_NODE=y # # Compile librte_cmdline diff --git a/config/defconfig_x86_64-default-linuxapp-icc b/config/defconfig_x86_64-default-linuxapp-icc index 86e656deba..fd2f21ceb2 100644 --- a/config/defconfig_x86_64-default-linuxapp-icc +++ b/config/defconfig_x86_64-default-linuxapp-icc @@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n CONFIG_RTE_LIBRTE_MALLOC=y CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M -CONFIG_RTE_MALLOC_PER_NUMA_NODE=y # # Compile librte_cmdline diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile index e2f929c4d3..32ceedde8c 100644 --- a/lib/librte_eal/common/Makefile +++ b/lib/librte_eal/common/Makefile @@ -38,7 +38,7 @@ INC += rte_log.h rte_memcpy.h rte_memory.h rte_memzone.h rte_pci.h INC += rte_pci_dev_ids.h rte_per_lcore.h rte_prefetch.h rte_random.h INC += rte_rwlock.h rte_spinlock.h rte_tailq.h rte_interrupts.h rte_alarm.h INC += rte_string_fns.h rte_cpuflags.h rte_version.h rte_tailq_elem.h -INC += rte_eal_memconfig.h +INC += rte_eal_memconfig.h rte_malloc_heap.h ifeq ($(CONFIG_RTE_INSECURE_FUNCTION_WARNING),y) INC += rte_warnings.h diff --git a/lib/librte_eal/common/include/rte_malloc_heap.h b/lib/librte_eal/common/include/rte_malloc_heap.h new file mode 100644 index 0000000000..3389f370ab --- /dev/null +++ b/lib/librte_eal/common/include/rte_malloc_heap.h @@ -0,0 +1,56 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2012 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _RTE_MALLOC_HEAP_H_ +#define _RTE_MALLOC_HEAP_H_ + +#include + +enum heap_state { + NOT_INITIALISED = 0, + INITIALISED +}; + +/** + * Structure to hold malloc heap + */ +struct malloc_heap { + enum heap_state volatile initialised; + unsigned numa_socket; + rte_spinlock_t lock; + struct malloc_elem * volatile free_head; + unsigned mz_count; +} __rte_cache_aligned; + +#endif /* _RTE_MALLOC_HEAP_H_ */ diff --git a/lib/librte_malloc/malloc_heap.h b/lib/librte_malloc/malloc_heap.h index 5ae1a9d014..b3250ba021 100644 --- a/lib/librte_malloc/malloc_heap.h +++ b/lib/librte_malloc/malloc_heap.h @@ -35,33 +35,28 @@ #ifndef MALLOC_HEAP_H_ #define MALLOC_HEAP_H_ -enum heap_state { - NOT_INITIALISED = 0, - INITIALISED -}; +#include +#include -struct malloc_heap { - enum heap_state initialised; - unsigned numa_socket; - volatile unsigned mz_count; - rte_spinlock_t lock; - struct malloc_elem * volatile free_head; -} __rte_cache_aligned; - -#define RTE_MALLOC_SOCKET_DEFAULT 0 +#ifdef __cplusplus +extern "C" { +#endif static inline unsigned malloc_get_numa_socket(void) { - unsigned malloc_socket = RTE_MALLOC_SOCKET_DEFAULT; - #ifdef RTE_MALLOC_PER_NUMA_NODE - malloc_socket = rte_socket_id(); - #endif - return malloc_socket; + return rte_socket_id(); } void * malloc_heap_alloc(struct malloc_heap *heap, const char *type, size_t size, unsigned align); +int +rte_eal_heap_memzone_init(void); + +#ifdef __cplusplus +} +#endif + #endif /* MALLOC_HEAP_H_ */ diff --git a/lib/librte_malloc/rte_malloc.c b/lib/librte_malloc/rte_malloc.c index 607da3c98d..7bf633d201 100644 --- a/lib/librte_malloc/rte_malloc.c +++ b/lib/librte_malloc/rte_malloc.c @@ -68,32 +68,66 @@ void rte_free(void *addr) } /* - * Allocate memory on default heap. + * Allocate memory on specified heap. */ void * -rte_malloc(const char *type, size_t size, unsigned align) +rte_malloc_socket(const char *type, size_t size, unsigned align, int socket) { - unsigned malloc_socket = malloc_get_numa_socket(); /* return NULL if size is 0 or alignment is not power-of-2 */ if (size == 0 || !rte_is_power_of_2(align)) return NULL; - return malloc_heap_alloc(&malloc_heap[malloc_socket], type, + + if (socket == SOCKET_ID_ANY) + socket = malloc_get_numa_socket(); + + /* Check socket parameter */ + if (socket >= RTE_MAX_NUMA_NODES) + return NULL; + + return malloc_heap_alloc(&malloc_heaps[socket], type, size, align == 0 ? 1 : align); } /* - * Allocate zero'd memory on default heap. + * Allocate memory on default heap. */ void * -rte_zmalloc(const char *type, size_t size, unsigned align) +rte_malloc(const char *type, size_t size, unsigned align) +{ + return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); +} + +/* + * Allocate zero'd memory on specified heap. + */ +void * +rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) { - void *ptr = rte_malloc(type, size, align); + void *ptr = rte_malloc_socket(type, size, align, socket); if (ptr != NULL) memset(ptr, 0, size); return ptr; } +/* + * Allocate zero'd memory on default heap. + */ +void * +rte_zmalloc(const char *type, size_t size, unsigned align) +{ + return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); +} + +/* + * Allocate zero'd memory on specified heap. + */ +void * +rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) +{ + return rte_zmalloc_socket(type, num * size, align, socket); +} + /* * Allocate zero'd memory on default heap. */ diff --git a/lib/librte_malloc/rte_malloc.h b/lib/librte_malloc/rte_malloc.h index 3804339de4..687ab16420 100644 --- a/lib/librte_malloc/rte_malloc.h +++ b/lib/librte_malloc/rte_malloc.h @@ -49,7 +49,8 @@ extern "C" { /** * This function allocates memory from the huge-page area of memory. The memory - * is not cleared. + * is not cleared. In NUMA systems, the memory allocated resides on the same + * NUMA socket as the core that calls this function. * * @param type * A string identifying the type of allocated objects (useful for debug @@ -74,7 +75,8 @@ rte_malloc(const char *type, size_t size, unsigned align); * Allocate zero'ed memory from the heap. * * Equivalent to rte_malloc() except that the memory zone is - * initialised with zeros. + * initialised with zeros. In NUMA systems, the memory allocated resides on the + * same NUMA socket as the core that calls this function. * * @param type * A string identifying the type of allocated objects (useful for debug @@ -97,7 +99,8 @@ rte_zmalloc(const char *type, size_t size, unsigned align); /** * Replacement function for calloc(), using huge-page memory. Memory area is - * initialised with zeros. + * initialised with zeros. In NUMA systems, the memory allocated resides on the + * same NUMA socket as the core that calls this function. * * @param type * A string identifying the type of allocated objects (useful for debug @@ -122,7 +125,8 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align); /** * Replacement function for realloc(), using huge-page memory. Reserved area - * memory is resized, preserving contents. + * memory is resized, preserving contents. In NUMA systems, the new area + * resides on the same NUMA socket as the old area. * * @param ptr * Pointer to already allocated memory @@ -142,6 +146,88 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align); void * rte_realloc(void *ptr, size_t size, unsigned align); +/** + * This function allocates memory from the huge-page area of memory. The memory + * is not cleared. + * + * @param type + * A string identifying the type of allocated objects (useful for debug + * purposes, such as identifying the cause of a memory leak). Can be NULL. + * @param size + * Size (in bytes) to be allocated. + * @param align + * If 0, the return is a pointer that is suitably aligned for any kind of + * variable (in the same manner as malloc()). + * Otherwise, the return is a pointer that is a multiple of *align*. In + * this case, it must be a power of two. (Minimum alignment is the + * cacheline size, i.e. 64-bytes) + * @param socket + * NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function + * will behave the same as rte_malloc(). + * @return + * - NULL on error. Not enough memory, or invalid arguments (size is 0, + * align is not a power of two). + * - Otherwise, the pointer to the allocated object. + */ +void * +rte_malloc_socket(const char *type, size_t size, unsigned align, int socket); + +/** + * Allocate zero'ed memory from the heap. + * + * Equivalent to rte_malloc() except that the memory zone is + * initialised with zeros. + * + * @param type + * A string identifying the type of allocated objects (useful for debug + * purposes, such as identifying the cause of a memory leak). Can be NULL. + * @param size + * Size (in bytes) to be allocated. + * @param align + * If 0, the return is a pointer that is suitably aligned for any kind of + * variable (in the same manner as malloc()). + * Otherwise, the return is a pointer that is a multiple of *align*. In + * this case, it must obviously be a power of two. (Minimum alignment is the + * cacheline size, i.e. 64-bytes) + * @param socket + * NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function + * will behave the same as rte_zmalloc(). + * @return + * - NULL on error. Not enough memory, or invalid arguments (size is 0, + * align is not a power of two). + * - Otherwise, the pointer to the allocated object. + */ +void * +rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket); + +/** + * Replacement function for calloc(), using huge-page memory. Memory area is + * initialised with zeros. + * + * @param type + * A string identifying the type of allocated objects (useful for debug + * purposes, such as identifying the cause of a memory leak). Can be NULL. + * @param num + * Number of elements to be allocated. + * @param size + * Size (in bytes) of a single element. + * @param align + * If 0, the return is a pointer that is suitably aligned for any kind of + * variable (in the same manner as malloc()). + * Otherwise, the return is a pointer that is a multiple of *align*. In + * this case, it must obviously be a power of two. (Minimum alignment is the + * cacheline size, i.e. 64-bytes) + * @param socket + * NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function + * will behave the same as rte_calloc(). + * @return + * - NULL on error. Not enough memory, or invalid arguments (size is 0, + * align is not a power of two). + * - Otherwise, the pointer to the allocated object. + */ +void * +rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket); + /** * Frees the memory space pointed to by the provided pointer. * @@ -193,6 +279,8 @@ rte_malloc_dump_stats(const char *type); /** * Set the maximum amount of allocated memory for this type. * + * This is not yet implemented + * * @param type * A string identifying the type of allocated objects. * @param max -- 2.20.1