#include <rte_common.h>
#include <rte_spinlock.h>
+#include "eal_memalloc.h"
#include "malloc_elem.h"
#include "malloc_heap.h"
* Initialize a general malloc_elem header structure
*/
void
-malloc_elem_init(struct malloc_elem *elem,
- struct malloc_heap *heap, const struct rte_memseg *ms, size_t size)
+malloc_elem_init(struct malloc_elem *elem, struct malloc_heap *heap,
+ struct rte_memseg_list *msl, size_t size)
{
elem->heap = heap;
- elem->ms = ms;
+ elem->msl = msl;
elem->prev = NULL;
elem->next = NULL;
memset(&elem->free_list, 0, sizeof(elem->free_list));
* so we just check the page addresses.
*/
static bool
-elem_check_phys_contig(const struct rte_memseg *ms __rte_unused,
+elem_check_phys_contig(const struct rte_memseg_list *msl,
void *start, size_t size)
{
- rte_iova_t cur, expected;
- void *start_page, *end_page, *cur_page;
- size_t pagesz;
-
- /* for hugepage memory or IOVA as VA, it's always contiguous */
- if (rte_eal_has_hugepages() || rte_eal_iova_mode() == RTE_IOVA_VA)
- return true;
-
- /* otherwise, check if start and end are within the same page */
- pagesz = getpagesize();
-
- start_page = RTE_PTR_ALIGN_FLOOR(start, pagesz);
- end_page = RTE_PTR_ALIGN_FLOOR(RTE_PTR_ADD(start, size - 1), pagesz);
-
- if (start_page == end_page)
- return true;
-
- /* if they are from different pages, check if they are contiguous */
-
- /* if we can't access physical addresses, assume non-contiguous */
- if (!rte_eal_using_phys_addrs())
- return false;
-
- /* skip first iteration */
- cur = rte_mem_virt2iova(start_page);
- expected = cur + pagesz;
- cur_page = RTE_PTR_ADD(start_page, pagesz);
-
- while (cur_page <= end_page) {
- cur = rte_mem_virt2iova(cur_page);
- if (cur != expected)
- return false;
- cur_page = RTE_PTR_ADD(cur_page, pagesz);
- expected += pagesz;
- }
- return true;
+ return eal_memalloc_is_contig(msl, start, size);
}
/*
* couldn't fit all data into one physically contiguous
* block, try again with lower addresses.
*/
- if (!elem_check_phys_contig(elem->ms,
+ if (!elem_check_phys_contig(elem->msl,
(void *)new_data_start,
new_data_size)) {
elem_size -= align;
const size_t old_elem_size = (uintptr_t)split_pt - (uintptr_t)elem;
const size_t new_elem_size = elem->size - old_elem_size;
- malloc_elem_init(split_pt, elem->heap, elem->ms, new_elem_size);
+ malloc_elem_init(split_pt, elem->heap, elem->msl, new_elem_size);
split_pt->prev = elem;
split_pt->next = next_elem;
if (next_elem)
return elem;
}
+/* assume all checks were already done */
+void
+malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len)
+{
+ struct malloc_elem *hide_start, *hide_end, *prev, *next;
+ size_t len_before, len_after;
+
+ hide_start = start;
+ hide_end = RTE_PTR_ADD(start, len);
+
+ prev = elem->prev;
+ next = elem->next;
+
+ /* we cannot do anything with non-adjacent elements */
+ if (next && next_elem_is_adjacent(elem)) {
+ len_after = RTE_PTR_DIFF(next, hide_end);
+ if (len_after >= MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
+ /* split after */
+ split_elem(elem, hide_end);
+
+ malloc_elem_free_list_insert(hide_end);
+ } else if (len_after >= MALLOC_ELEM_HEADER_LEN) {
+ /* shrink current element */
+ elem->size -= len_after;
+ memset(hide_end, 0, sizeof(*hide_end));
+
+ /* copy next element's data to our pad */
+ memcpy(hide_end, next, sizeof(*hide_end));
+
+ /* pad next element */
+ next->state = ELEM_PAD;
+ next->pad = len_after;
+ next->size -= len_after;
+
+ /* next element busy, would've been merged otherwise */
+ hide_end->pad = len_after;
+ hide_end->size += len_after;
+
+ /* adjust pointers to point to our new pad */
+ if (next->next)
+ next->next->prev = hide_end;
+ elem->next = hide_end;
+ } else if (len_after > 0) {
+ RTE_LOG(ERR, EAL, "Unaligned element, heap is probably corrupt\n");
+ return;
+ }
+ }
+
+ /* we cannot do anything with non-adjacent elements */
+ if (prev && prev_elem_is_adjacent(elem)) {
+ len_before = RTE_PTR_DIFF(hide_start, elem);
+ if (len_before >= MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
+ /* split before */
+ split_elem(elem, hide_start);
+
+ prev = elem;
+ elem = hide_start;
+
+ malloc_elem_free_list_insert(prev);
+ } else if (len_before > 0) {
+ /*
+ * unlike with elements after current, here we don't
+ * need to pad elements, but rather just increase the
+ * size of previous element, copy the old header and set
+ * up trailer.
+ */
+ void *trailer = RTE_PTR_ADD(prev,
+ prev->size - MALLOC_ELEM_TRAILER_LEN);
+
+ memcpy(hide_start, elem, sizeof(*elem));
+ hide_start->size = len;
+
+ prev->size += len_before;
+ set_trailer(prev);
+
+ /* update pointers */
+ prev->next = hide_start;
+ if (next)
+ next->prev = hide_start;
+
+ /* erase old trailer */
+ memset(trailer, 0, MALLOC_ELEM_TRAILER_LEN);
+ /* erase old header */
+ memset(elem, 0, sizeof(*elem));
+
+ elem = hide_start;
+ }
+ }
+
+ remove_elem(elem);
+}
+
/*
* attempt to resize a malloc_elem by expanding into any free space
* immediately after it in memory.
git.droids-corp.org / dpdk.git / blobdiff