1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
10 #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
12 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */
18 ELEM_PAD /* element is a padding-only header */
22 struct malloc_heap *heap;
23 struct malloc_elem *volatile prev;
24 /**< points to prev elem in memseg */
25 struct malloc_elem *volatile next;
26 /**< points to next elem in memseg */
27 LIST_ENTRY(malloc_elem) free_list;
28 /**< list of free elements in heap */
29 struct rte_memseg_list *msl;
30 volatile enum elem_state state;
33 struct malloc_elem *orig_elem;
35 #ifdef RTE_MALLOC_DEBUG
36 uint64_t header_cookie; /* Cookie marking start of data */
37 /* trailer cookie at start + size */
39 #ifdef RTE_MALLOC_ASAN
41 uint64_t asan_cookie[2]; /* must be next to header_cookie */
43 } __rte_cache_aligned;
45 static const unsigned int MALLOC_ELEM_HEADER_LEN = sizeof(struct malloc_elem);
47 #ifndef RTE_MALLOC_DEBUG
48 #ifdef RTE_MALLOC_ASAN
49 static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
51 static const unsigned int MALLOC_ELEM_TRAILER_LEN;
54 /* dummy function - just check if pointer is non-null */
56 malloc_elem_cookies_ok(const struct malloc_elem *elem){ return elem != NULL; }
58 /* dummy function - no header if malloc_debug is not enabled */
60 set_header(struct malloc_elem *elem __rte_unused){ }
62 /* dummy function - no trailer if malloc_debug is not enabled */
64 set_trailer(struct malloc_elem *elem __rte_unused){ }
68 static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
70 #define MALLOC_HEADER_COOKIE 0xbadbadbadadd2e55ULL /**< Header cookie. */
71 #define MALLOC_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
73 /* define macros to make referencing the header and trailer cookies easier */
74 #define MALLOC_ELEM_TRAILER(elem) (*((uint64_t*)RTE_PTR_ADD(elem, \
75 elem->size - MALLOC_ELEM_TRAILER_LEN)))
76 #define MALLOC_ELEM_HEADER(elem) (elem->header_cookie)
79 set_header(struct malloc_elem *elem)
82 MALLOC_ELEM_HEADER(elem) = MALLOC_HEADER_COOKIE;
86 set_trailer(struct malloc_elem *elem)
89 MALLOC_ELEM_TRAILER(elem) = MALLOC_TRAILER_COOKIE;
92 /* check that the header and trailer cookies are set correctly */
94 malloc_elem_cookies_ok(const struct malloc_elem *elem)
96 return elem != NULL &&
97 MALLOC_ELEM_HEADER(elem) == MALLOC_HEADER_COOKIE &&
98 MALLOC_ELEM_TRAILER(elem) == MALLOC_TRAILER_COOKIE;
103 #define MALLOC_ELEM_OVERHEAD (MALLOC_ELEM_HEADER_LEN + MALLOC_ELEM_TRAILER_LEN)
105 #ifdef RTE_MALLOC_ASAN
107 #ifdef RTE_ARCH_X86_64
108 #define ASAN_SHADOW_OFFSET 0x00007fff8000
111 #define ASAN_SHADOW_GRAIN_SIZE 8
112 #define ASAN_MEM_FREE_FLAG 0xfd
113 #define ASAN_MEM_REDZONE_FLAG 0xfa
114 #define ASAN_SHADOW_SCALE 3
116 #define ASAN_MEM_SHIFT(mem) ((void *)((uintptr_t)(mem) >> ASAN_SHADOW_SCALE))
117 #define ASAN_MEM_TO_SHADOW(mem) \
118 RTE_PTR_ADD(ASAN_MEM_SHIFT(mem), ASAN_SHADOW_OFFSET)
120 #if defined(__clang__)
121 #define __rte_no_asan __attribute__((no_sanitize("address", "hwaddress")))
123 #define __rte_no_asan __attribute__((no_sanitize_address))
128 asan_set_shadow(void *addr, char val)
134 asan_set_zone(void *ptr, size_t len, uint32_t val)
138 size_t zone_len = len / ASAN_SHADOW_GRAIN_SIZE;
139 if (len % ASAN_SHADOW_GRAIN_SIZE != 0)
142 for (i = 0; i < zone_len; i++) {
143 offset = i * ASAN_SHADOW_GRAIN_SIZE;
144 shadow = ASAN_MEM_TO_SHADOW((uintptr_t)ptr + offset);
145 asan_set_shadow(shadow, val);
150 * When the memory is released, the release mark is
151 * set in the corresponding range of the shadow area.
154 asan_set_freezone(void *ptr, size_t size)
156 asan_set_zone(ptr, size, ASAN_MEM_FREE_FLAG);
160 * When the memory is allocated, memory state must set as accessible.
163 asan_clear_alloczone(struct malloc_elem *elem)
165 asan_set_zone((void *)elem, elem->size, 0x0);
169 asan_clear_split_alloczone(struct malloc_elem *elem)
171 void *ptr = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
172 asan_set_zone(ptr, MALLOC_ELEM_OVERHEAD, 0x0);
176 * When the memory is allocated, the memory boundary is
177 * marked in the corresponding range of the shadow area.
178 * Requirement: redzone >= 16, is a power of two.
181 asan_set_redzone(struct malloc_elem *elem, size_t user_size)
183 uintptr_t head_redzone;
184 uintptr_t tail_redzone;
190 if (elem->state != ELEM_PAD)
191 elem = RTE_PTR_ADD(elem, elem->pad);
193 elem->user_size = user_size;
195 /* Set mark before the start of the allocated memory */
196 head_redzone = (uintptr_t)RTE_PTR_ADD(elem,
197 MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE);
198 front_shadow = ASAN_MEM_TO_SHADOW(head_redzone);
199 asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG);
200 front_shadow = ASAN_MEM_TO_SHADOW(head_redzone
201 - ASAN_SHADOW_GRAIN_SIZE);
202 asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG);
204 /* Set mark after the end of the allocated memory */
205 tail_redzone = (uintptr_t)RTE_PTR_ADD(elem,
206 MALLOC_ELEM_HEADER_LEN
208 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone);
209 val = (tail_redzone % ASAN_SHADOW_GRAIN_SIZE);
210 val = (val == 0) ? ASAN_MEM_REDZONE_FLAG : val;
211 asan_set_shadow(tail_shadow, val);
212 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone
213 + ASAN_SHADOW_GRAIN_SIZE);
214 asan_set_shadow(tail_shadow, ASAN_MEM_REDZONE_FLAG);
219 * When the memory is released, the mark of the memory boundary
220 * in the corresponding range of the shadow area is cleared.
221 * Requirement: redzone >= 16, is a power of two.
224 asan_clear_redzone(struct malloc_elem *elem)
226 uintptr_t head_redzone;
227 uintptr_t tail_redzone;
232 elem = RTE_PTR_ADD(elem, elem->pad);
234 /* Clear mark before the start of the allocated memory */
235 head_redzone = (uintptr_t)RTE_PTR_ADD(elem,
236 MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE);
237 head_shadow = ASAN_MEM_TO_SHADOW(head_redzone);
238 asan_set_shadow(head_shadow, 0x00);
239 head_shadow = ASAN_MEM_TO_SHADOW(head_redzone
240 - ASAN_SHADOW_GRAIN_SIZE);
241 asan_set_shadow(head_shadow, 0x00);
243 /* Clear mark after the end of the allocated memory */
244 tail_redzone = (uintptr_t)RTE_PTR_ADD(elem,
245 MALLOC_ELEM_HEADER_LEN + elem->user_size);
246 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone);
247 asan_set_shadow(tail_shadow, 0x00);
248 tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone
249 + ASAN_SHADOW_GRAIN_SIZE);
250 asan_set_shadow(tail_shadow, 0x00);
255 old_malloc_size(struct malloc_elem *elem)
257 if (elem->state != ELEM_PAD)
258 elem = RTE_PTR_ADD(elem, elem->pad);
260 return elem->user_size;
263 #else /* !RTE_MALLOC_ASAN */
265 #define __rte_no_asan
268 asan_set_freezone(void *ptr __rte_unused, size_t size __rte_unused) { }
271 asan_clear_alloczone(struct malloc_elem *elem __rte_unused) { }
274 asan_clear_split_alloczone(struct malloc_elem *elem __rte_unused) { }
277 asan_set_redzone(struct malloc_elem *elem __rte_unused,
278 size_t user_size __rte_unused) { }
281 asan_clear_redzone(struct malloc_elem *elem __rte_unused) { }
284 old_malloc_size(struct malloc_elem *elem)
286 return elem->size - elem->pad - MALLOC_ELEM_OVERHEAD;
288 #endif /* !RTE_MALLOC_ASAN */
291 * Given a pointer to the start of a memory block returned by malloc, get
292 * the actual malloc_elem header for that block.
294 static inline struct malloc_elem *
295 malloc_elem_from_data(const void *data)
300 struct malloc_elem *elem = RTE_PTR_SUB(data, MALLOC_ELEM_HEADER_LEN);
301 if (!malloc_elem_cookies_ok(elem))
303 return elem->state != ELEM_PAD ? elem: RTE_PTR_SUB(elem, elem->pad);
307 * initialise a malloc_elem header
310 malloc_elem_init(struct malloc_elem *elem,
311 struct malloc_heap *heap,
312 struct rte_memseg_list *msl,
314 struct malloc_elem *orig_elem,
318 malloc_elem_insert(struct malloc_elem *elem);
321 * return true if the current malloc_elem can hold a block of data
322 * of the requested size and with the requested alignment
325 malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
326 unsigned int align, size_t bound, bool contig);
329 * reserve a block of data in an existing malloc_elem. If the malloc_elem
330 * is much larger than the data block requested, we split the element in two.
333 malloc_elem_alloc(struct malloc_elem *elem, size_t size,
334 unsigned int align, size_t bound, bool contig);
337 * free a malloc_elem block by adding it to the free list. If the
338 * blocks either immediately before or immediately after newly freed block
339 * are also free, the blocks are merged together.
342 malloc_elem_free(struct malloc_elem *elem);
345 malloc_elem_join_adjacent_free(struct malloc_elem *elem);
348 * attempt to resize a malloc_elem by expanding into any free space
349 * immediately after it in memory.
352 malloc_elem_resize(struct malloc_elem *elem, size_t size);
355 malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len);
358 malloc_elem_free_list_remove(struct malloc_elem *elem);
361 * dump contents of malloc elem to a file.
364 malloc_elem_dump(const struct malloc_elem *elem, FILE *f);
367 * Given an element size, compute its freelist index.
370 malloc_elem_free_list_index(size_t size);
373 * Add element to its heap's free list.
376 malloc_elem_free_list_insert(struct malloc_elem *elem);
379 * Find biggest IOVA-contiguous zone within an element with specified alignment.
382 malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align);
384 #endif /* MALLOC_ELEM_H_ */