1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
10 #include <sys/queue.h>
12 #include <rte_memory.h>
14 #include <rte_eal_memconfig.h>
15 #include <rte_launch.h>
16 #include <rte_per_lcore.h>
17 #include <rte_lcore.h>
18 #include <rte_common.h>
19 #include <rte_string_fns.h>
20 #include <rte_spinlock.h>
21 #include <rte_memcpy.h>
22 #include <rte_atomic.h>
24 #include "malloc_elem.h"
25 #include "malloc_heap.h"
28 check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
30 unsigned check_flag = 0;
32 if (!(flags & ~RTE_MEMZONE_SIZE_HINT_ONLY))
35 switch (hugepage_sz) {
37 check_flag = RTE_MEMZONE_256KB;
40 check_flag = RTE_MEMZONE_2MB;
43 check_flag = RTE_MEMZONE_16MB;
46 check_flag = RTE_MEMZONE_256MB;
49 check_flag = RTE_MEMZONE_512MB;
52 check_flag = RTE_MEMZONE_1GB;
55 check_flag = RTE_MEMZONE_4GB;
58 check_flag = RTE_MEMZONE_16GB;
61 return check_flag & flags;
65 * Expand the heap with a memseg.
66 * This reserves the zone and sets a dummy malloc_elem header at the end
67 * to prevent overflow. The rest of the zone is added to free list as a single
71 malloc_heap_add_memseg(const struct rte_memseg *ms, void *arg __rte_unused)
73 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
74 struct malloc_elem *start_elem;
75 struct rte_memseg *found_ms;
76 struct malloc_heap *heap;
80 heap = &mcfg->malloc_heaps[ms->socket_id];
82 /* ms is const, so find it */
83 ms_idx = ms - mcfg->memseg;
84 found_ms = &mcfg->memseg[ms_idx];
86 start_elem = (struct malloc_elem *)found_ms->addr;
87 elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
89 malloc_elem_init(start_elem, heap, found_ms, elem_size);
90 malloc_elem_insert(start_elem);
91 malloc_elem_free_list_insert(start_elem);
93 heap->total_size += elem_size;
99 * Iterates through the freelist for a heap to find a free element
100 * which can store data of the required size and with the requested alignment.
101 * If size is 0, find the biggest available elem.
102 * Returns null on failure, or pointer to element on success.
104 static struct malloc_elem *
105 find_suitable_element(struct malloc_heap *heap, size_t size,
106 unsigned int flags, size_t align, size_t bound, bool contig)
109 struct malloc_elem *elem, *alt_elem = NULL;
111 for (idx = malloc_elem_free_list_index(size);
112 idx < RTE_HEAP_NUM_FREELISTS; idx++) {
113 for (elem = LIST_FIRST(&heap->free_head[idx]);
114 !!elem; elem = LIST_NEXT(elem, free_list)) {
115 if (malloc_elem_can_hold(elem, size, align, bound,
117 if (check_hugepage_sz(flags, elem->ms->hugepage_sz))
119 if (alt_elem == NULL)
125 if ((alt_elem != NULL) && (flags & RTE_MEMZONE_SIZE_HINT_ONLY))
132 * Main function to allocate a block of memory from the heap.
133 * It locks the free list, scans it, and adds a new memseg if the
134 * scan fails. Once the new memseg is added, it re-scans and should return
135 * the new element after releasing the lock.
138 malloc_heap_alloc(struct malloc_heap *heap,
139 const char *type __attribute__((unused)), size_t size, unsigned flags,
140 size_t align, size_t bound, bool contig)
142 struct malloc_elem *elem;
144 size = RTE_CACHE_LINE_ROUNDUP(size);
145 align = RTE_CACHE_LINE_ROUNDUP(align);
147 rte_spinlock_lock(&heap->lock);
149 elem = find_suitable_element(heap, size, flags, align, bound, contig);
151 elem = malloc_elem_alloc(elem, size, align, bound, contig);
152 /* increase heap's count of allocated elements */
155 rte_spinlock_unlock(&heap->lock);
157 return elem == NULL ? NULL : (void *)(&elem[1]);
161 malloc_heap_free(struct malloc_elem *elem)
163 struct malloc_heap *heap;
164 struct malloc_elem *ret;
166 if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
169 /* elem may be merged with previous element, so keep heap address */
172 rte_spinlock_lock(&(heap->lock));
174 ret = malloc_elem_free(elem);
176 rte_spinlock_unlock(&(heap->lock));
178 return ret != NULL ? 0 : -1;
182 malloc_heap_resize(struct malloc_elem *elem, size_t size)
186 if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
189 rte_spinlock_lock(&(elem->heap->lock));
191 ret = malloc_elem_resize(elem, size);
193 rte_spinlock_unlock(&(elem->heap->lock));
199 * Function to retrieve data for heap on given socket
202 malloc_heap_get_stats(struct malloc_heap *heap,
203 struct rte_malloc_socket_stats *socket_stats)
206 struct malloc_elem *elem;
208 rte_spinlock_lock(&heap->lock);
210 /* Initialise variables for heap */
211 socket_stats->free_count = 0;
212 socket_stats->heap_freesz_bytes = 0;
213 socket_stats->greatest_free_size = 0;
215 /* Iterate through free list */
216 for (idx = 0; idx < RTE_HEAP_NUM_FREELISTS; idx++) {
217 for (elem = LIST_FIRST(&heap->free_head[idx]);
218 !!elem; elem = LIST_NEXT(elem, free_list))
220 socket_stats->free_count++;
221 socket_stats->heap_freesz_bytes += elem->size;
222 if (elem->size > socket_stats->greatest_free_size)
223 socket_stats->greatest_free_size = elem->size;
226 /* Get stats on overall heap and allocated memory on this heap */
227 socket_stats->heap_totalsz_bytes = heap->total_size;
228 socket_stats->heap_allocsz_bytes = (socket_stats->heap_totalsz_bytes -
229 socket_stats->heap_freesz_bytes);
230 socket_stats->alloc_count = heap->alloc_count;
232 rte_spinlock_unlock(&heap->lock);
237 * Function to retrieve data for heap on given socket
240 malloc_heap_dump(struct malloc_heap *heap, FILE *f)
242 struct malloc_elem *elem;
244 rte_spinlock_lock(&heap->lock);
246 fprintf(f, "Heap size: 0x%zx\n", heap->total_size);
247 fprintf(f, "Heap alloc count: %u\n", heap->alloc_count);
251 malloc_elem_dump(elem, f);
255 rte_spinlock_unlock(&heap->lock);
259 rte_eal_malloc_heap_init(void)
261 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
266 rte_memseg_walk(malloc_heap_add_memseg, NULL);