1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
10 #include <sys/queue.h>
12 #include <rte_memory.h>
14 #include <rte_eal_memconfig.h>
15 #include <rte_launch.h>
16 #include <rte_per_lcore.h>
17 #include <rte_lcore.h>
18 #include <rte_common.h>
19 #include <rte_string_fns.h>
20 #include <rte_spinlock.h>
21 #include <rte_memcpy.h>
22 #include <rte_atomic.h>
24 #include "malloc_elem.h"
25 #include "malloc_heap.h"
28 check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
30 unsigned check_flag = 0;
32 if (!(flags & ~RTE_MEMZONE_SIZE_HINT_ONLY))
35 switch (hugepage_sz) {
37 check_flag = RTE_MEMZONE_256KB;
40 check_flag = RTE_MEMZONE_2MB;
43 check_flag = RTE_MEMZONE_16MB;
46 check_flag = RTE_MEMZONE_256MB;
49 check_flag = RTE_MEMZONE_512MB;
52 check_flag = RTE_MEMZONE_1GB;
55 check_flag = RTE_MEMZONE_4GB;
58 check_flag = RTE_MEMZONE_16GB;
61 return check_flag & flags;
65 * Expand the heap with a memseg.
66 * This reserves the zone and sets a dummy malloc_elem header at the end
67 * to prevent overflow. The rest of the zone is added to free list as a single
71 malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
73 struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr;
74 const size_t elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
76 malloc_elem_init(start_elem, heap, ms, elem_size);
77 malloc_elem_insert(start_elem);
78 malloc_elem_free_list_insert(start_elem);
80 heap->total_size += elem_size;
84 * Iterates through the freelist for a heap to find a free element
85 * which can store data of the required size and with the requested alignment.
86 * If size is 0, find the biggest available elem.
87 * Returns null on failure, or pointer to element on success.
89 static struct malloc_elem *
90 find_suitable_element(struct malloc_heap *heap, size_t size,
91 unsigned int flags, size_t align, size_t bound, bool contig)
94 struct malloc_elem *elem, *alt_elem = NULL;
96 for (idx = malloc_elem_free_list_index(size);
97 idx < RTE_HEAP_NUM_FREELISTS; idx++) {
98 for (elem = LIST_FIRST(&heap->free_head[idx]);
99 !!elem; elem = LIST_NEXT(elem, free_list)) {
100 if (malloc_elem_can_hold(elem, size, align, bound,
102 if (check_hugepage_sz(flags, elem->ms->hugepage_sz))
104 if (alt_elem == NULL)
110 if ((alt_elem != NULL) && (flags & RTE_MEMZONE_SIZE_HINT_ONLY))
117 * Main function to allocate a block of memory from the heap.
118 * It locks the free list, scans it, and adds a new memseg if the
119 * scan fails. Once the new memseg is added, it re-scans and should return
120 * the new element after releasing the lock.
123 malloc_heap_alloc(struct malloc_heap *heap,
124 const char *type __attribute__((unused)), size_t size, unsigned flags,
125 size_t align, size_t bound, bool contig)
127 struct malloc_elem *elem;
129 size = RTE_CACHE_LINE_ROUNDUP(size);
130 align = RTE_CACHE_LINE_ROUNDUP(align);
132 rte_spinlock_lock(&heap->lock);
134 elem = find_suitable_element(heap, size, flags, align, bound, contig);
136 elem = malloc_elem_alloc(elem, size, align, bound, contig);
137 /* increase heap's count of allocated elements */
140 rte_spinlock_unlock(&heap->lock);
142 return elem == NULL ? NULL : (void *)(&elem[1]);
146 malloc_heap_free(struct malloc_elem *elem)
148 struct malloc_heap *heap;
149 struct malloc_elem *ret;
151 if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
154 /* elem may be merged with previous element, so keep heap address */
157 rte_spinlock_lock(&(heap->lock));
159 ret = malloc_elem_free(elem);
161 rte_spinlock_unlock(&(heap->lock));
163 return ret != NULL ? 0 : -1;
167 malloc_heap_resize(struct malloc_elem *elem, size_t size)
171 if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
174 rte_spinlock_lock(&(elem->heap->lock));
176 ret = malloc_elem_resize(elem, size);
178 rte_spinlock_unlock(&(elem->heap->lock));
184 * Function to retrieve data for heap on given socket
187 malloc_heap_get_stats(struct malloc_heap *heap,
188 struct rte_malloc_socket_stats *socket_stats)
191 struct malloc_elem *elem;
193 rte_spinlock_lock(&heap->lock);
195 /* Initialise variables for heap */
196 socket_stats->free_count = 0;
197 socket_stats->heap_freesz_bytes = 0;
198 socket_stats->greatest_free_size = 0;
200 /* Iterate through free list */
201 for (idx = 0; idx < RTE_HEAP_NUM_FREELISTS; idx++) {
202 for (elem = LIST_FIRST(&heap->free_head[idx]);
203 !!elem; elem = LIST_NEXT(elem, free_list))
205 socket_stats->free_count++;
206 socket_stats->heap_freesz_bytes += elem->size;
207 if (elem->size > socket_stats->greatest_free_size)
208 socket_stats->greatest_free_size = elem->size;
211 /* Get stats on overall heap and allocated memory on this heap */
212 socket_stats->heap_totalsz_bytes = heap->total_size;
213 socket_stats->heap_allocsz_bytes = (socket_stats->heap_totalsz_bytes -
214 socket_stats->heap_freesz_bytes);
215 socket_stats->alloc_count = heap->alloc_count;
217 rte_spinlock_unlock(&heap->lock);
222 * Function to retrieve data for heap on given socket
225 malloc_heap_dump(struct malloc_heap *heap, FILE *f)
227 struct malloc_elem *elem;
229 rte_spinlock_lock(&heap->lock);
231 fprintf(f, "Heap size: 0x%zx\n", heap->total_size);
232 fprintf(f, "Heap alloc count: %u\n", heap->alloc_count);
236 malloc_elem_dump(elem, f);
240 rte_spinlock_unlock(&heap->lock);
244 rte_eal_malloc_heap_init(void)
246 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
248 struct rte_memseg *ms;
253 for (ms = &mcfg->memseg[0], ms_cnt = 0;
254 (ms_cnt < RTE_MAX_MEMSEG) && (ms->len > 0);
256 malloc_heap_add_memseg(&mcfg->malloc_heaps[ms->socket_id], ms);