1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
10 #include <sys/queue.h>
12 #include <rte_memory.h>
14 #include <rte_eal_memconfig.h>
15 #include <rte_launch.h>
16 #include <rte_per_lcore.h>
17 #include <rte_lcore.h>
18 #include <rte_common.h>
19 #include <rte_string_fns.h>
20 #include <rte_spinlock.h>
21 #include <rte_memcpy.h>
22 #include <rte_atomic.h>
24 #include "eal_internal_cfg.h"
25 #include "malloc_elem.h"
26 #include "malloc_heap.h"
29 check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
31 unsigned check_flag = 0;
33 if (!(flags & ~RTE_MEMZONE_SIZE_HINT_ONLY))
36 switch (hugepage_sz) {
38 check_flag = RTE_MEMZONE_256KB;
41 check_flag = RTE_MEMZONE_2MB;
44 check_flag = RTE_MEMZONE_16MB;
47 check_flag = RTE_MEMZONE_256MB;
50 check_flag = RTE_MEMZONE_512MB;
53 check_flag = RTE_MEMZONE_1GB;
56 check_flag = RTE_MEMZONE_4GB;
59 check_flag = RTE_MEMZONE_16GB;
62 return check_flag & flags;
66 * Expand the heap with a memory area.
68 static struct malloc_elem *
69 malloc_heap_add_memory(struct malloc_heap *heap, struct rte_memseg_list *msl,
70 void *start, size_t len)
72 struct malloc_elem *elem = start;
74 malloc_elem_init(elem, heap, msl, len);
76 malloc_elem_insert(elem);
78 elem = malloc_elem_join_adjacent_free(elem);
80 malloc_elem_free_list_insert(elem);
82 heap->total_size += len;
88 malloc_add_seg(const struct rte_memseg_list *msl,
89 const struct rte_memseg *ms, size_t len, void *arg __rte_unused)
91 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
92 struct rte_memseg_list *found_msl;
93 struct malloc_heap *heap;
96 heap = &mcfg->malloc_heaps[msl->socket_id];
98 /* msl is const, so find it */
99 msl_idx = msl - mcfg->memsegs;
100 found_msl = &mcfg->memsegs[msl_idx];
102 if (msl_idx < 0 || msl_idx >= RTE_MAX_MEMSEG_LISTS)
105 malloc_heap_add_memory(heap, found_msl, ms->addr, len);
107 RTE_LOG(DEBUG, EAL, "Added %zuM to heap on socket %i\n", len >> 20,
113 * Iterates through the freelist for a heap to find a free element
114 * which can store data of the required size and with the requested alignment.
115 * If size is 0, find the biggest available elem.
116 * Returns null on failure, or pointer to element on success.
118 static struct malloc_elem *
119 find_suitable_element(struct malloc_heap *heap, size_t size,
120 unsigned int flags, size_t align, size_t bound, bool contig)
123 struct malloc_elem *elem, *alt_elem = NULL;
125 for (idx = malloc_elem_free_list_index(size);
126 idx < RTE_HEAP_NUM_FREELISTS; idx++) {
127 for (elem = LIST_FIRST(&heap->free_head[idx]);
128 !!elem; elem = LIST_NEXT(elem, free_list)) {
129 if (malloc_elem_can_hold(elem, size, align, bound,
131 if (check_hugepage_sz(flags,
134 if (alt_elem == NULL)
140 if ((alt_elem != NULL) && (flags & RTE_MEMZONE_SIZE_HINT_ONLY))
147 * Main function to allocate a block of memory from the heap.
148 * It locks the free list, scans it, and adds a new memseg if the
149 * scan fails. Once the new memseg is added, it re-scans and should return
150 * the new element after releasing the lock.
153 malloc_heap_alloc(struct malloc_heap *heap,
154 const char *type __attribute__((unused)), size_t size, unsigned flags,
155 size_t align, size_t bound, bool contig)
157 struct malloc_elem *elem;
159 size = RTE_CACHE_LINE_ROUNDUP(size);
160 align = RTE_CACHE_LINE_ROUNDUP(align);
162 rte_spinlock_lock(&heap->lock);
164 elem = find_suitable_element(heap, size, flags, align, bound, contig);
166 elem = malloc_elem_alloc(elem, size, align, bound, contig);
167 /* increase heap's count of allocated elements */
170 rte_spinlock_unlock(&heap->lock);
172 return elem == NULL ? NULL : (void *)(&elem[1]);
176 malloc_heap_free(struct malloc_elem *elem)
178 struct malloc_heap *heap;
179 struct malloc_elem *ret;
181 if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
184 /* elem may be merged with previous element, so keep heap address */
187 rte_spinlock_lock(&(heap->lock));
189 ret = malloc_elem_free(elem);
191 rte_spinlock_unlock(&(heap->lock));
193 return ret != NULL ? 0 : -1;
197 malloc_heap_resize(struct malloc_elem *elem, size_t size)
201 if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
204 rte_spinlock_lock(&(elem->heap->lock));
206 ret = malloc_elem_resize(elem, size);
208 rte_spinlock_unlock(&(elem->heap->lock));
214 * Function to retrieve data for heap on given socket
217 malloc_heap_get_stats(struct malloc_heap *heap,
218 struct rte_malloc_socket_stats *socket_stats)
221 struct malloc_elem *elem;
223 rte_spinlock_lock(&heap->lock);
225 /* Initialise variables for heap */
226 socket_stats->free_count = 0;
227 socket_stats->heap_freesz_bytes = 0;
228 socket_stats->greatest_free_size = 0;
230 /* Iterate through free list */
231 for (idx = 0; idx < RTE_HEAP_NUM_FREELISTS; idx++) {
232 for (elem = LIST_FIRST(&heap->free_head[idx]);
233 !!elem; elem = LIST_NEXT(elem, free_list))
235 socket_stats->free_count++;
236 socket_stats->heap_freesz_bytes += elem->size;
237 if (elem->size > socket_stats->greatest_free_size)
238 socket_stats->greatest_free_size = elem->size;
241 /* Get stats on overall heap and allocated memory on this heap */
242 socket_stats->heap_totalsz_bytes = heap->total_size;
243 socket_stats->heap_allocsz_bytes = (socket_stats->heap_totalsz_bytes -
244 socket_stats->heap_freesz_bytes);
245 socket_stats->alloc_count = heap->alloc_count;
247 rte_spinlock_unlock(&heap->lock);
252 * Function to retrieve data for heap on given socket
255 malloc_heap_dump(struct malloc_heap *heap, FILE *f)
257 struct malloc_elem *elem;
259 rte_spinlock_lock(&heap->lock);
261 fprintf(f, "Heap size: 0x%zx\n", heap->total_size);
262 fprintf(f, "Heap alloc count: %u\n", heap->alloc_count);
266 malloc_elem_dump(elem, f);
270 rte_spinlock_unlock(&heap->lock);
274 rte_eal_malloc_heap_init(void)
276 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
281 /* add all IOVA-contiguous areas to the heap */
282 return rte_memseg_contig_walk(malloc_add_seg, NULL);