#include <stdbool.h>
+#include <rte_common.h>
+
#define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
/* dummy definition of struct so we can use pointers to it in malloc_elem struct */
LIST_ENTRY(malloc_elem) free_list;
/**< list of free elements in heap */
struct rte_memseg_list *msl;
- volatile enum elem_state state;
+ /** Element state, @c dirty and @c pad validity depends on it. */
+ /* An extra bit is needed to represent enum elem_state as signed int. */
+ enum elem_state state : 3;
+ /** If state == ELEM_FREE: the memory is not filled with zeroes. */
+ uint32_t dirty : 1;
+ /** Reserved for future use. */
+ uint32_t reserved : 28;
uint32_t pad;
size_t size;
struct malloc_elem *orig_elem;
uint64_t header_cookie; /* Cookie marking start of data */
/* trailer cookie at start + size */
#endif
+#ifdef RTE_MALLOC_ASAN
+ size_t user_size;
+ uint64_t asan_cookie[2]; /* must be next to header_cookie */
+#endif
} __rte_cache_aligned;
+static const unsigned int MALLOC_ELEM_HEADER_LEN = sizeof(struct malloc_elem);
+
#ifndef RTE_MALLOC_DEBUG
-static const unsigned MALLOC_ELEM_TRAILER_LEN = 0;
+#ifdef RTE_MALLOC_ASAN
+static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
+#else
+static const unsigned int MALLOC_ELEM_TRAILER_LEN;
+#endif
/* dummy function - just check if pointer is non-null */
static inline int
#else
-static const unsigned MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
+static const unsigned int MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
#define MALLOC_HEADER_COOKIE 0xbadbadbadadd2e55ULL /**< Header cookie. */
#define MALLOC_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
#endif
-static const unsigned MALLOC_ELEM_HEADER_LEN = sizeof(struct malloc_elem);
#define MALLOC_ELEM_OVERHEAD (MALLOC_ELEM_HEADER_LEN + MALLOC_ELEM_TRAILER_LEN)
+#ifdef RTE_MALLOC_ASAN
+
+/*
+ * ASAN_SHADOW_OFFSET should match to the corresponding
+ * value defined in gcc/libsanitizer/asan/asan_mapping.h
+ */
+#ifdef RTE_ARCH_X86_64
+#define ASAN_SHADOW_OFFSET 0x00007fff8000
+#elif defined(RTE_ARCH_ARM64)
+#define ASAN_SHADOW_OFFSET 0x001000000000
+#elif defined(RTE_ARCH_PPC_64)
+#define ASAN_SHADOW_OFFSET 0x020000000000
+#endif
+
+#define ASAN_SHADOW_GRAIN_SIZE 8
+#define ASAN_MEM_FREE_FLAG 0xfd
+#define ASAN_MEM_REDZONE_FLAG 0xfa
+#define ASAN_SHADOW_SCALE 3
+
+#define ASAN_MEM_SHIFT(mem) ((void *)((uintptr_t)(mem) >> ASAN_SHADOW_SCALE))
+#define ASAN_MEM_TO_SHADOW(mem) \
+ RTE_PTR_ADD(ASAN_MEM_SHIFT(mem), ASAN_SHADOW_OFFSET)
+
+__rte_no_asan
+static inline void
+asan_set_shadow(void *addr, char val)
+{
+ *(char *)addr = val;
+}
+
+static inline void
+asan_set_zone(void *ptr, size_t len, uint32_t val)
+{
+ size_t offset, i;
+ void *shadow;
+ size_t zone_len = len / ASAN_SHADOW_GRAIN_SIZE;
+ if (len % ASAN_SHADOW_GRAIN_SIZE != 0)
+ zone_len += 1;
+
+ for (i = 0; i < zone_len; i++) {
+ offset = i * ASAN_SHADOW_GRAIN_SIZE;
+ shadow = ASAN_MEM_TO_SHADOW((uintptr_t)ptr + offset);
+ asan_set_shadow(shadow, val);
+ }
+}
+
+/*
+ * When the memory is released, the release mark is
+ * set in the corresponding range of the shadow area.
+ */
+static inline void
+asan_set_freezone(void *ptr, size_t size)
+{
+ asan_set_zone(ptr, size, ASAN_MEM_FREE_FLAG);
+}
+
+/*
+ * When the memory is allocated, memory state must set as accessible.
+ */
+static inline void
+asan_clear_alloczone(struct malloc_elem *elem)
+{
+ asan_set_zone((void *)elem, elem->size, 0x0);
+}
+
+static inline void
+asan_clear_split_alloczone(struct malloc_elem *elem)
+{
+ void *ptr = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
+ asan_set_zone(ptr, MALLOC_ELEM_OVERHEAD, 0x0);
+}
+
+/*
+ * When the memory is allocated, the memory boundary is
+ * marked in the corresponding range of the shadow area.
+ * Requirement: redzone >= 16, is a power of two.
+ */
+static inline void
+asan_set_redzone(struct malloc_elem *elem, size_t user_size)
+{
+ uintptr_t head_redzone;
+ uintptr_t tail_redzone;
+ void *front_shadow;
+ void *tail_shadow;
+ uint32_t val;
+
+ if (elem != NULL) {
+ if (elem->state != ELEM_PAD)
+ elem = RTE_PTR_ADD(elem, elem->pad);
+
+ elem->user_size = user_size;
+
+ /* Set mark before the start of the allocated memory */
+ head_redzone = (uintptr_t)RTE_PTR_ADD(elem,
+ MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE);
+ front_shadow = ASAN_MEM_TO_SHADOW(head_redzone);
+ asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG);
+ front_shadow = ASAN_MEM_TO_SHADOW(head_redzone
+ - ASAN_SHADOW_GRAIN_SIZE);
+ asan_set_shadow(front_shadow, ASAN_MEM_REDZONE_FLAG);
+
+ /* Set mark after the end of the allocated memory */
+ tail_redzone = (uintptr_t)RTE_PTR_ADD(elem,
+ MALLOC_ELEM_HEADER_LEN
+ + elem->user_size);
+ tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone);
+ val = (tail_redzone % ASAN_SHADOW_GRAIN_SIZE);
+ val = (val == 0) ? ASAN_MEM_REDZONE_FLAG : val;
+ asan_set_shadow(tail_shadow, val);
+ tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone
+ + ASAN_SHADOW_GRAIN_SIZE);
+ asan_set_shadow(tail_shadow, ASAN_MEM_REDZONE_FLAG);
+ }
+}
+
+/*
+ * When the memory is released, the mark of the memory boundary
+ * in the corresponding range of the shadow area is cleared.
+ * Requirement: redzone >= 16, is a power of two.
+ */
+static inline void
+asan_clear_redzone(struct malloc_elem *elem)
+{
+ uintptr_t head_redzone;
+ uintptr_t tail_redzone;
+ void *head_shadow;
+ void *tail_shadow;
+
+ if (elem != NULL) {
+ elem = RTE_PTR_ADD(elem, elem->pad);
+
+ /* Clear mark before the start of the allocated memory */
+ head_redzone = (uintptr_t)RTE_PTR_ADD(elem,
+ MALLOC_ELEM_HEADER_LEN - ASAN_SHADOW_GRAIN_SIZE);
+ head_shadow = ASAN_MEM_TO_SHADOW(head_redzone);
+ asan_set_shadow(head_shadow, 0x00);
+ head_shadow = ASAN_MEM_TO_SHADOW(head_redzone
+ - ASAN_SHADOW_GRAIN_SIZE);
+ asan_set_shadow(head_shadow, 0x00);
+
+ /* Clear mark after the end of the allocated memory */
+ tail_redzone = (uintptr_t)RTE_PTR_ADD(elem,
+ MALLOC_ELEM_HEADER_LEN + elem->user_size);
+ tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone);
+ asan_set_shadow(tail_shadow, 0x00);
+ tail_shadow = ASAN_MEM_TO_SHADOW(tail_redzone
+ + ASAN_SHADOW_GRAIN_SIZE);
+ asan_set_shadow(tail_shadow, 0x00);
+ }
+}
+
+static inline size_t
+old_malloc_size(struct malloc_elem *elem)
+{
+ if (elem->state != ELEM_PAD)
+ elem = RTE_PTR_ADD(elem, elem->pad);
+
+ return elem->user_size;
+}
+
+#else /* !RTE_MALLOC_ASAN */
+
+static inline void
+asan_set_zone(void *ptr __rte_unused, size_t len __rte_unused,
+ uint32_t val __rte_unused) { }
+
+static inline void
+asan_set_freezone(void *ptr __rte_unused, size_t size __rte_unused) { }
+
+static inline void
+asan_clear_alloczone(struct malloc_elem *elem __rte_unused) { }
+
+static inline void
+asan_clear_split_alloczone(struct malloc_elem *elem __rte_unused) { }
+
+static inline void
+asan_set_redzone(struct malloc_elem *elem __rte_unused,
+ size_t user_size __rte_unused) { }
+
+static inline void
+asan_clear_redzone(struct malloc_elem *elem __rte_unused) { }
+
+static inline size_t
+old_malloc_size(struct malloc_elem *elem)
+{
+ return elem->size - elem->pad - MALLOC_ELEM_OVERHEAD;
+}
+#endif /* !RTE_MALLOC_ASAN */
+
/*
* Given a pointer to the start of a memory block returned by malloc, get
* the actual malloc_elem header for that block.
struct rte_memseg_list *msl,
size_t size,
struct malloc_elem *orig_elem,
- size_t orig_size);
+ size_t orig_size,
+ bool dirty);
void
malloc_elem_insert(struct malloc_elem *elem);
git.droids-corp.org / dpdk.git / blobdiff