#include <rte_common.h>
#include <rte_spinlock.h>
+#include "eal_memalloc.h"
#include "malloc_elem.h"
#include "malloc_heap.h"
-#define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
-
/*
* Initialize a general malloc_elem header structure
*/
struct malloc_elem *prev_elem, *next_elem;
struct malloc_heap *heap = elem->heap;
+ /* first and last elements must be both NULL or both non-NULL */
+ if ((heap->first == NULL) != (heap->last == NULL)) {
+ RTE_LOG(ERR, EAL, "Heap is probably corrupt\n");
+ return;
+ }
+
if (heap->first == NULL && heap->last == NULL) {
/* if empty heap */
heap->first = elem;
* so we just check the page addresses.
*/
static bool
-elem_check_phys_contig(const struct rte_memseg_list *msl __rte_unused,
+elem_check_phys_contig(const struct rte_memseg_list *msl,
void *start, size_t size)
{
- rte_iova_t cur, expected;
- void *start_page, *end_page, *cur_page;
- size_t pagesz;
-
- /* for hugepage memory or IOVA as VA, it's always contiguous */
- if (rte_eal_has_hugepages() || rte_eal_iova_mode() == RTE_IOVA_VA)
- return true;
-
- /* otherwise, check if start and end are within the same page */
- pagesz = getpagesize();
-
- start_page = RTE_PTR_ALIGN_FLOOR(start, pagesz);
- end_page = RTE_PTR_ALIGN_FLOOR(RTE_PTR_ADD(start, size - 1), pagesz);
-
- if (start_page == end_page)
- return true;
-
- /* if they are from different pages, check if they are contiguous */
-
- /* if we can't access physical addresses, assume non-contiguous */
- if (!rte_eal_using_phys_addrs())
- return false;
-
- /* skip first iteration */
- cur = rte_mem_virt2iova(start_page);
- expected = cur + pagesz;
- cur_page = RTE_PTR_ADD(start_page, pagesz);
-
- while (cur_page <= end_page) {
- cur = rte_mem_virt2iova(cur_page);
- if (cur != expected)
- return false;
- cur_page = RTE_PTR_ADD(cur_page, pagesz);
- expected += pagesz;
- }
- return true;
+ return eal_memalloc_is_contig(msl, start, size);
}
/*
malloc_elem_free_list_insert(elem);
+ elem->pad = 0;
+
/* decrease heap's count of allocated elements */
elem->heap->alloc_count--;
return elem;
}
+/* assume all checks were already done */
+void
+malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len)
+{
+ struct malloc_elem *hide_start, *hide_end, *prev, *next;
+ size_t len_before, len_after;
+
+ hide_start = start;
+ hide_end = RTE_PTR_ADD(start, len);
+
+ prev = elem->prev;
+ next = elem->next;
+
+ /* we cannot do anything with non-adjacent elements */
+ if (next && next_elem_is_adjacent(elem)) {
+ len_after = RTE_PTR_DIFF(next, hide_end);
+ if (len_after >= MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
+ /* split after */
+ split_elem(elem, hide_end);
+
+ malloc_elem_free_list_insert(hide_end);
+ } else if (len_after > 0) {
+ RTE_LOG(ERR, EAL, "Unaligned element, heap is probably corrupt\n");
+ return;
+ }
+ }
+
+ /* we cannot do anything with non-adjacent elements */
+ if (prev && prev_elem_is_adjacent(elem)) {
+ len_before = RTE_PTR_DIFF(hide_start, elem);
+ if (len_before >= MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
+ /* split before */
+ split_elem(elem, hide_start);
+
+ prev = elem;
+ elem = hide_start;
+
+ malloc_elem_free_list_insert(prev);
+ } else if (len_before > 0) {
+ RTE_LOG(ERR, EAL, "Unaligned element, heap is probably corrupt\n");
+ return;
+ }
+ }
+
+ remove_elem(elem);
+}
+
/*
* attempt to resize a malloc_elem by expanding into any free space
* immediately after it in memory.