malloc: add NUMA-aware realloc function

Currently, rte_realloc will not respect original allocation's
NUMA node when memory cannot be resized, and there is no
NUMA-aware equivalent of rte_realloc. This patch adds such a function.

The new API will ensure that reallocated memory stays on
requested NUMA node, as well as allow moving allocated memory
to a different NUMA node.

Signed-off-by: Tomasz Jozwiak <tomaszx.jozwiak@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

diff --git a/app/test/test_malloc.c b/app/test/test_malloc.c
index 6b6c6fe..f0e608c 100644 (file)
--- a/app/test/test_malloc.c
+++ b/app/test/test_malloc.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2019 Intel Corporation
  */
 
 #include <stdio.h>
@@ -26,6 +26,13 @@
 
 #define N 10000
 
+
+static int
+is_mem_on_socket(int32_t socket);
+
+static int32_t
+addr_to_socket(void *addr);
+
 /*
  * Malloc
  * ======
@@ -542,7 +549,49 @@ test_realloc(void)
                return -1;
        }
        rte_free(ptr12);
-       return 0;
+
+       /* check realloc_socket part */
+       int32_t socket_count = 0, socket_allocated, socket;
+       int ret = -1;
+       size_t size = 1024;
+
+       ptr1 = NULL;
+       for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
+               if (is_mem_on_socket(socket)) {
+                       int j = 2;
+
+                       socket_count++;
+                       while (j--) {
+                               /* j == 1 -> resizing */
+                               ptr2 = rte_realloc_socket(ptr1, size,
+                                                         RTE_CACHE_LINE_SIZE,
+                                                         socket);
+                               if (ptr2 == NULL) {
+                                       printf("NULL pointer returned from rte_realloc_socket\n");
+                                       goto end;
+                               }
+
+                               ptr1 = ptr2;
+                               socket_allocated = addr_to_socket(ptr2);
+                               if (socket_allocated != socket) {
+                                       printf("Requested socket (%d) doesn't mach allocated one (%d)\n",
+                                              socket, socket_allocated);
+                                       goto end;
+                               }
+                               size += RTE_CACHE_LINE_SIZE;
+                       }
+               }
+       }
+
+       /* Print warnign if only a single socket, but don't fail the test */
+       if (socket_count < 2)
+               printf("WARNING: realloc_socket test needs memory on multiple sockets!\n");
+
+       ret = 0;
+end:
+       rte_free(ptr1);
+
+       return ret;
 }
 
 static int

diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h
index 54a1246..5fbde4e 100644 (file)
--- a/lib/librte_eal/common/include/rte_malloc.h
+++ b/lib/librte_eal/common/include/rte_malloc.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2019 Intel Corporation
  */
 
 #ifndef _RTE_MALLOC_H_
@@ -129,7 +129,32 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align);
  *   - Otherwise, the pointer to the reallocated memory.
  */
 void *
-rte_realloc(void *ptr, size_t size, unsigned align);
+rte_realloc(void *ptr, size_t size, unsigned int align);
+
+/**
+ * Replacement function for realloc(), using huge-page memory. Reserved area
+ * memory is resized, preserving contents. In NUMA systems, the new area
+ * resides on requested NUMA socket.
+ *
+ * @param ptr
+ *   Pointer to already allocated memory
+ * @param size
+ *   Size (in bytes) of new area. If this is 0, memory is freed.
+ * @param align
+ *   If 0, the return is a pointer that is suitably aligned for any kind of
+ *   variable (in the same manner as malloc()).
+ *   Otherwise, the return is a pointer that is a multiple of *align*. In
+ *   this case, it must obviously be a power of two. (Minimum alignment is the
+ *   cacheline size, i.e. 64-bytes)
+ * @param socket
+ *   NUMA socket to allocate memory on.
+ * @return
+ *   - NULL on error. Not enough memory, or invalid arguments (size is 0,
+ *     align is not a power of two).
+ *   - Otherwise, the pointer to the reallocated memory.
+ */
+void * __rte_experimental
+rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket);
 
 /**
  * This function allocates memory from the huge-page area of memory. The memory

diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c
index b39de3c..eebd067 100644 (file)
--- a/lib/librte_eal/common/rte_malloc.c
+++ b/lib/librte_eal/common/rte_malloc.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2019 Intel Corporation
  */
 
 #include <stdint.h>
@@ -105,13 +105,13 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align)
 }
 
 /*
- * Resize allocated memory.
+ * Resize allocated memory on specified heap.
  */
 void *
-rte_realloc(void *ptr, size_t size, unsigned align)
+rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket)
 {
        if (ptr == NULL)
-               return rte_malloc(NULL, size, align);
+               return rte_malloc_socket(NULL, size, align, socket);
 
        struct malloc_elem *elem = malloc_elem_from_data(ptr);
        if (elem == NULL) {
@@ -120,14 +120,21 @@ rte_realloc(void *ptr, size_t size, unsigned align)
        }
 
        size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align);
-       /* check alignment matches first, and if ok, see if we can resize block */
-       if (RTE_PTR_ALIGN(ptr,align) == ptr &&
+
+       /* check requested socket id and alignment matches first, and if ok,
+        * see if we can resize block
+        */
+       if ((socket == SOCKET_ID_ANY ||
+            (unsigned int)socket == elem->heap->socket_id) &&
+                       RTE_PTR_ALIGN(ptr, align) == ptr &&
                        malloc_heap_resize(elem, size) == 0)
                return ptr;
 
-       /* either alignment is off, or we have no room to expand,
-        * so move data. */
-       void *new_ptr = rte_malloc(NULL, size, align);
+       /* either requested socket id doesn't match, alignment is off
+        * or we have no room to expand,
+        * so move the data.
+        */
+       void *new_ptr = rte_malloc_socket(NULL, size, align, socket);
        if (new_ptr == NULL)
                return NULL;
        const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD;
@@ -137,6 +144,15 @@ rte_realloc(void *ptr, size_t size, unsigned align)
        return new_ptr;
 }
 
+/*
+ * Resize allocated memory.
+ */
+void *
+rte_realloc(void *ptr, size_t size, unsigned int align)
+{
+       return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY);
+}
+
 int
 rte_malloc_validate(const void *ptr, size_t *size)
 {

diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9..935e918 100644 (file)
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -359,6 +359,7 @@ EXPERIMENTAL {
        rte_mp_request_async;
        rte_mp_sendmsg;
        rte_option_register;
+       rte_realloc_socket;
        rte_service_lcore_attr_get;
        rte_service_lcore_attr_reset_all;
        rte_service_may_be_active;