1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2019 Intel Corporation
11 #include <sys/queue.h>
13 #include <rte_common.h>
14 #include <rte_memory.h>
15 #include <rte_eal_memconfig.h>
16 #include <rte_per_lcore.h>
17 #include <rte_launch.h>
19 #include <rte_lcore.h>
20 #include <rte_malloc.h>
21 #include <rte_cycles.h>
22 #include <rte_random.h>
23 #include <rte_string_fns.h>
31 is_mem_on_socket(int32_t socket);
34 addr_to_socket(void *addr);
40 * Allocate some dynamic memory from heap (3 areas). Check that areas
41 * don't overlap and that alignment constraints match. This test is
42 * done many times on different lcores simultaneously.
45 /* Test if memory overlaps: return 1 if true, or 0 if false. */
47 is_memory_overlap(void *p1, size_t len1, void *p2, size_t len2)
49 unsigned long ptr1 = (unsigned long)p1;
50 unsigned long ptr2 = (unsigned long)p2;
52 if (ptr2 >= ptr1 && (ptr2 - ptr1) < len1)
54 else if (ptr2 < ptr1 && (ptr1 - ptr2) < len2)
60 is_aligned(void *p, int align)
62 unsigned long addr = (unsigned long)p;
63 unsigned mask = align - 1;
71 test_align_overlap_per_lcore(__attribute__((unused)) void *arg)
73 const unsigned align1 = 8,
77 void *p1 = NULL, *p2 = NULL, *p3 = NULL;
80 for (i = 0; i < N; i++) {
81 p1 = rte_zmalloc("dummy", 1000, align1);
83 printf("rte_zmalloc returned NULL (i=%u)\n", i);
87 for(j = 0; j < 1000 ; j++) {
88 if( *(char *)p1 != 0) {
89 printf("rte_zmalloc didn't zero the allocated memory\n");
93 p2 = rte_malloc("dummy", 1000, align2);
95 printf("rte_malloc returned NULL (i=%u)\n", i);
100 p3 = rte_malloc("dummy", 1000, align3);
102 printf("rte_malloc returned NULL (i=%u)\n", i);
108 if (is_memory_overlap(p1, 1000, p2, 1000)) {
109 printf("p1 and p2 overlaps\n");
112 if (is_memory_overlap(p2, 1000, p3, 1000)) {
113 printf("p2 and p3 overlaps\n");
116 if (is_memory_overlap(p1, 1000, p3, 1000)) {
117 printf("p1 and p3 overlaps\n");
120 if (!is_aligned(p1, align1)) {
121 printf("p1 is not aligned\n");
124 if (!is_aligned(p2, align2)) {
125 printf("p2 is not aligned\n");
128 if (!is_aligned(p3, align3)) {
129 printf("p3 is not aligned\n");
136 rte_malloc_dump_stats(stdout, "dummy");
142 test_reordered_free_per_lcore(__attribute__((unused)) void *arg)
144 const unsigned align1 = 8,
151 for (i = 0; i < 30; i++) {
152 p1 = rte_zmalloc("dummy", 1000, align1);
154 printf("rte_zmalloc returned NULL (i=%u)\n", i);
158 for(j = 0; j < 1000 ; j++) {
159 if( *(char *)p1 != 0) {
160 printf("rte_zmalloc didn't zero the allocated memory\n");
164 /* use calloc to allocate 1000 16-byte items this time */
165 p2 = rte_calloc("dummy", 1000, 16, align2);
166 /* for third request use regular malloc again */
167 p3 = rte_malloc("dummy", 1000, align3);
169 printf("rte_malloc returned NULL (i=%u)\n", i);
173 if (is_memory_overlap(p1, 1000, p2, 1000)) {
174 printf("p1 and p2 overlaps\n");
177 if (is_memory_overlap(p2, 1000, p3, 1000)) {
178 printf("p2 and p3 overlaps\n");
181 if (is_memory_overlap(p1, 1000, p3, 1000)) {
182 printf("p1 and p3 overlaps\n");
185 if (!is_aligned(p1, align1)) {
186 printf("p1 is not aligned\n");
189 if (!is_aligned(p2, align2)) {
190 printf("p2 is not aligned\n");
193 if (!is_aligned(p3, align3)) {
194 printf("p3 is not aligned\n");
197 /* try freeing in every possible order */
231 rte_malloc_dump_stats(stdout, "dummy");
236 /* test function inside the malloc lib*/
238 test_str_to_size(void)
244 {{ "5G", (uint64_t)5 * 1024 * 1024 *1024 },
245 {"0x20g", (uint64_t)0x20 * 1024 * 1024 *1024},
246 {"10M", 10 * 1024 * 1024},
247 {"050m", 050 * 1024 * 1024},
253 {"-1", 0}, /* negative values return 0 */
256 {"18446744073709551616", 0} /* ULLONG_MAX + 1 == out of range*/
259 for (i = 0; i < sizeof(test_values)/sizeof(test_values[0]); i++)
260 if (rte_str_to_size(test_values[i].str) != test_values[i].value)
266 test_multi_alloc_statistics(void)
269 struct rte_malloc_socket_stats pre_stats, post_stats ,first_stats, second_stats;
274 /* Dynamically calculate the overhead by allocating one cacheline and
275 * then comparing what was allocated from the heap.
277 rte_malloc_get_socket_stats(socket, &pre_stats);
279 void *dummy = rte_malloc_socket(NULL, RTE_CACHE_LINE_SIZE, 0, socket);
283 rte_malloc_get_socket_stats(socket, &post_stats);
285 /* after subtracting cache line, remainder is overhead */
286 overhead = post_stats.heap_allocsz_bytes - pre_stats.heap_allocsz_bytes;
287 overhead -= RTE_CACHE_LINE_SIZE;
291 /* Now start the real tests */
292 rte_malloc_get_socket_stats(socket, &pre_stats);
294 void *p1 = rte_malloc_socket("stats", size , align, socket);
298 rte_malloc_dump_stats(stdout, "stats");
300 rte_malloc_get_socket_stats(socket,&post_stats);
301 /* Check statistics reported are correct */
302 /* All post stats should be equal to pre stats after alloc freed */
303 if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) &&
304 (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) &&
305 (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&&
306 (post_stats.alloc_count!=pre_stats.alloc_count)&&
307 (post_stats.free_count!=pre_stats.free_count)) {
308 printf("Malloc statistics are incorrect - freed alloc\n");
311 /* Check two consecutive allocations */
314 rte_malloc_get_socket_stats(socket,&pre_stats);
315 void *p2 = rte_malloc_socket("add", size ,align, socket);
318 rte_malloc_get_socket_stats(socket,&first_stats);
320 void *p3 = rte_malloc_socket("add2", size,align, socket);
324 rte_malloc_get_socket_stats(socket,&second_stats);
329 /* After freeing both allocations check stats return to original */
330 rte_malloc_get_socket_stats(socket, &post_stats);
332 if(second_stats.heap_totalsz_bytes != first_stats.heap_totalsz_bytes) {
333 printf("Incorrect heap statistics: Total size \n");
336 /* Check allocated size is equal to two additions plus overhead */
337 if(second_stats.heap_allocsz_bytes !=
338 size + overhead + first_stats.heap_allocsz_bytes) {
339 printf("Incorrect heap statistics: Allocated size \n");
342 /* Check that allocation count increments correctly i.e. +1 */
343 if (second_stats.alloc_count != first_stats.alloc_count + 1) {
344 printf("Incorrect heap statistics: Allocated count \n");
348 if (second_stats.free_count != first_stats.free_count){
349 printf("Incorrect heap statistics: Free count \n");
353 /* Make sure that we didn't touch our greatest chunk: 2 * 11M) */
354 if (post_stats.greatest_free_size != pre_stats.greatest_free_size) {
355 printf("Incorrect heap statistics: Greatest free size \n");
358 /* Free size must equal the original free size minus the new allocation*/
359 if (first_stats.heap_freesz_bytes <= second_stats.heap_freesz_bytes) {
360 printf("Incorrect heap statistics: Free size \n");
364 if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) &&
365 (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) &&
366 (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&&
367 (post_stats.alloc_count!=pre_stats.alloc_count)&&
368 (post_stats.free_count!=pre_stats.free_count)) {
369 printf("Malloc statistics are incorrect - freed alloc\n");
376 test_rte_malloc_type_limits(void)
378 /* The type-limits functionality is not yet implemented,
379 * so always return 0 no matter what the retval.
381 const char *typename = "limit_test";
382 rte_malloc_set_limit(typename, 64 * 1024);
383 rte_malloc_dump_stats(stdout, typename);
390 const char hello_str[] = "Hello, world!";
391 const unsigned size1 = 1024;
392 const unsigned size2 = size1 + 1024;
393 const unsigned size3 = size2;
394 const unsigned size4 = size3 + 1024;
396 /* test data is the same even if element is moved*/
397 char *ptr1 = rte_zmalloc(NULL, size1, RTE_CACHE_LINE_SIZE);
399 printf("NULL pointer returned from rte_zmalloc\n");
402 strlcpy(ptr1, hello_str, size1);
403 char *ptr2 = rte_realloc(ptr1, size2, RTE_CACHE_LINE_SIZE);
406 printf("NULL pointer returned from rte_realloc\n");
410 printf("unexpected - ptr1 == ptr2\n");
412 if (strcmp(ptr2, hello_str) != 0){
413 printf("Error - lost data from pointed area\n");
418 for (i = strnlen(hello_str, sizeof(hello_str)); i < size1; i++)
420 printf("Bad data in realloc\n");
424 /* now allocate third element, free the second
425 * and resize third. It should not move. (ptr1 is now invalid)
427 char *ptr3 = rte_zmalloc(NULL, size3, RTE_CACHE_LINE_SIZE);
429 printf("NULL pointer returned from rte_zmalloc\n");
433 for (i = 0; i < size3; i++)
435 printf("Bad data in zmalloc\n");
441 /* first resize to half the size of the freed block */
442 char *ptr4 = rte_realloc(ptr3, size4, RTE_CACHE_LINE_SIZE);
444 printf("NULL pointer returned from rte_realloc\n");
449 printf("Unexpected - ptr4 != ptr3\n");
453 /* now resize again to the full size of the freed block */
454 ptr4 = rte_realloc(ptr3, size3 + size2 + size1, RTE_CACHE_LINE_SIZE);
456 printf("Unexpected - ptr4 != ptr3 on second resize\n");
462 /* now try a resize to a smaller size, see if it works */
463 const unsigned size5 = 1024;
464 const unsigned size6 = size5 / 2;
465 char *ptr5 = rte_malloc(NULL, size5, RTE_CACHE_LINE_SIZE);
467 printf("NULL pointer returned from rte_malloc\n");
470 char *ptr6 = rte_realloc(ptr5, size6, RTE_CACHE_LINE_SIZE);
472 printf("NULL pointer returned from rte_realloc\n");
477 printf("Error, resizing to a smaller size moved data\n");
483 /* check for behaviour changing alignment */
484 const unsigned size7 = 1024;
485 const unsigned orig_align = RTE_CACHE_LINE_SIZE;
486 unsigned new_align = RTE_CACHE_LINE_SIZE * 2;
487 char *ptr7 = rte_malloc(NULL, size7, orig_align);
489 printf("NULL pointer returned from rte_malloc\n");
492 /* calc an alignment we don't already have */
493 while(RTE_PTR_ALIGN(ptr7, new_align) == ptr7)
495 char *ptr8 = rte_realloc(ptr7, size7, new_align);
497 printf("NULL pointer returned from rte_realloc\n");
501 if (RTE_PTR_ALIGN(ptr8, new_align) != ptr8){
502 printf("Failure to re-align data\n");
508 /* test behaviour when there is a free block after current one,
509 * but its not big enough
511 unsigned size9 = 1024, size10 = 1024;
512 unsigned size11 = size9 + size10 + 256;
513 char *ptr9 = rte_malloc(NULL, size9, RTE_CACHE_LINE_SIZE);
515 printf("NULL pointer returned from rte_malloc\n");
518 char *ptr10 = rte_malloc(NULL, size10, RTE_CACHE_LINE_SIZE);
520 printf("NULL pointer returned from rte_malloc\n");
524 char *ptr11 = rte_realloc(ptr10, size11, RTE_CACHE_LINE_SIZE);
526 printf("NULL pointer returned from rte_realloc\n");
531 printf("Error, unexpected that realloc has not created new buffer\n");
537 /* check we don't crash if we pass null to realloc
538 * We should get a malloc of the size requested*/
539 const size_t size12 = 1024;
541 char *ptr12 = rte_realloc(NULL, size12, RTE_CACHE_LINE_SIZE);
543 printf("NULL pointer returned from rte_realloc\n");
546 if (rte_malloc_validate(ptr12, &size12_check) < 0 ||
547 size12_check != size12){
553 /* check realloc_socket part */
554 int32_t socket_count = 0, socket_allocated, socket;
559 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
560 if (is_mem_on_socket(socket)) {
565 /* j == 1 -> resizing */
566 ptr2 = rte_realloc_socket(ptr1, size,
570 printf("NULL pointer returned from rte_realloc_socket\n");
575 socket_allocated = addr_to_socket(ptr2);
576 if (socket_allocated != socket) {
577 printf("Requested socket (%d) doesn't mach allocated one (%d)\n",
578 socket, socket_allocated);
581 size += RTE_CACHE_LINE_SIZE;
586 /* Print warnign if only a single socket, but don't fail the test */
587 if (socket_count < 2)
588 printf("WARNING: realloc_socket test needs memory on multiple sockets!\n");
598 test_random_alloc_free(void *_ __attribute__((unused)))
601 struct mem_list *next;
607 rte_srand((unsigned)rte_rdtsc());
609 for (i = 0; i < N; i++){
610 unsigned free_mem = 0;
611 size_t allocated_size;
613 const unsigned mem_size = sizeof(struct mem_list) + \
614 rte_rand() % (64 * 1024);
615 const unsigned align = 1 << (rte_rand() % 12); /* up to 4k alignment */
616 struct mem_list *entry = rte_malloc(NULL,
620 if (RTE_PTR_ALIGN(entry, align)!= entry)
622 if (rte_malloc_validate(entry, &allocated_size) == -1
623 || allocated_size < mem_size)
625 memset(entry->data, rte_lcore_id(),
626 mem_size - sizeof(*entry));
627 entry->next = list_head;
628 if (rte_malloc_validate(entry, NULL) == -1)
633 /* switch to freeing the memory with a 20% probability */
634 free_mem = ((rte_rand() % 10) >= 8);
637 struct mem_list *entry = list_head;
638 list_head = list_head->next;
642 printf("Lcore %u allocated/freed %u blocks\n", rte_lcore_id(), count);
646 #define err_return() do { \
647 printf("%s: %d - Error\n", __func__, __LINE__); \
652 test_rte_malloc_validate(void)
654 const size_t request_size = 1024;
655 size_t allocated_size;
656 char *data_ptr = rte_malloc(NULL, request_size, RTE_CACHE_LINE_SIZE);
657 #ifdef RTE_MALLOC_DEBUG
659 char *over_write_vals = NULL;
662 if (data_ptr == NULL) {
663 printf("%s: %d - Allocation error\n", __func__, __LINE__);
667 /* check that a null input returns -1 */
668 if (rte_malloc_validate(NULL, NULL) != -1)
671 /* check that we get ok on a valid pointer */
672 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
675 /* check that the returned size is ok */
676 if (allocated_size < request_size)
679 #ifdef RTE_MALLOC_DEBUG
681 /****** change the header to be bad */
683 over_write_vals = (char *)((uintptr_t)data_ptr - sizeof(save_buf));
684 /* first save the data as a backup before overwriting it */
685 memcpy(save_buf, over_write_vals, sizeof(save_buf));
686 memset(over_write_vals, 1, sizeof(save_buf));
687 /* then run validate */
688 retval = rte_malloc_validate(data_ptr, NULL);
689 /* finally restore the data again */
690 memcpy(over_write_vals, save_buf, sizeof(save_buf));
691 /* check we previously had an error */
695 /* check all ok again */
696 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
699 /**** change the trailer to be bad */
700 over_write_vals = (char *)((uintptr_t)data_ptr + allocated_size);
701 /* first save the data as a backup before overwriting it */
702 memcpy(save_buf, over_write_vals, sizeof(save_buf));
703 memset(over_write_vals, 1, sizeof(save_buf));
704 /* then run validate */
705 retval = rte_malloc_validate(data_ptr, NULL);
706 /* finally restore the data again */
707 memcpy(over_write_vals, save_buf, sizeof(save_buf));
711 /* check all ok again */
712 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
726 test_zero_aligned_alloc(void)
728 char *p1 = rte_malloc(NULL,1024, 0);
731 if (!rte_is_aligned(p1, RTE_CACHE_LINE_SIZE))
738 if (p1) rte_free(p1);
743 test_malloc_bad_params(void)
745 const char *type = NULL;
747 unsigned align = RTE_CACHE_LINE_SIZE;
749 /* rte_malloc expected to return null with inappropriate size */
750 char *bad_ptr = rte_malloc(type, size, align);
754 /* rte_malloc expected to return null with inappropriate alignment */
758 bad_ptr = rte_malloc(type, size, align);
765 /* clean up pointer */
772 check_socket_mem(const struct rte_memseg_list *msl, void *arg)
774 int32_t *socket = arg;
779 return *socket == msl->socket_id;
782 /* Check if memory is available on a specific socket */
784 is_mem_on_socket(int32_t socket)
786 return rte_memseg_list_walk(check_socket_mem, &socket);
791 * Find what socket a memory address is on. Only works for addresses within
792 * memsegs, not heap or stack...
795 addr_to_socket(void * addr)
797 const struct rte_memseg *ms = rte_mem_virt2memseg(addr, NULL);
798 return ms == NULL ? -1 : ms->socket_id;
802 /* Test using rte_[c|m|zm]alloc_socket() on a specific socket */
804 test_alloc_single_socket(int32_t socket)
806 const char *type = NULL;
807 const size_t size = 10;
808 const unsigned align = 0;
810 int32_t desired_socket = (socket == SOCKET_ID_ANY) ?
811 (int32_t)rte_socket_id() : socket;
813 /* Test rte_calloc_socket() */
814 mem = rte_calloc_socket(type, size, sizeof(char), align, socket);
817 if (addr_to_socket(mem) != desired_socket) {
823 /* Test rte_malloc_socket() */
824 mem = rte_malloc_socket(type, size, align, socket);
827 if (addr_to_socket(mem) != desired_socket) {
832 /* Test rte_zmalloc_socket() */
833 mem = rte_zmalloc_socket(type, size, align, socket);
836 if (addr_to_socket(mem) != desired_socket) {
846 test_alloc_socket(void)
848 unsigned socket_count = 0;
851 if (test_alloc_single_socket(SOCKET_ID_ANY) < 0)
854 for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
855 if (is_mem_on_socket(i)) {
857 if (test_alloc_single_socket(i) < 0) {
858 printf("Fail: rte_malloc_socket(..., %u) did not succeed\n",
864 if (test_alloc_single_socket(i) == 0) {
865 printf("Fail: rte_malloc_socket(..., %u) succeeded\n",
872 /* Print warnign if only a single socket, but don't fail the test */
873 if (socket_count < 2) {
874 printf("WARNING: alloc_socket test needs memory on multiple sockets!\n");
886 if (test_str_to_size() < 0){
887 printf("test_str_to_size() failed\n");
890 else printf("test_str_to_size() passed\n");
892 if (test_zero_aligned_alloc() < 0){
893 printf("test_zero_aligned_alloc() failed\n");
896 else printf("test_zero_aligned_alloc() passed\n");
898 if (test_malloc_bad_params() < 0){
899 printf("test_malloc_bad_params() failed\n");
902 else printf("test_malloc_bad_params() passed\n");
904 if (test_realloc() < 0){
905 printf("test_realloc() failed\n");
908 else printf("test_realloc() passed\n");
910 /*----------------------------*/
911 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
912 rte_eal_remote_launch(test_align_overlap_per_lcore, NULL, lcore_id);
915 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
916 if (rte_eal_wait_lcore(lcore_id) < 0)
920 printf("test_align_overlap_per_lcore() failed\n");
923 else printf("test_align_overlap_per_lcore() passed\n");
925 /*----------------------------*/
926 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
927 rte_eal_remote_launch(test_reordered_free_per_lcore, NULL, lcore_id);
930 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
931 if (rte_eal_wait_lcore(lcore_id) < 0)
935 printf("test_reordered_free_per_lcore() failed\n");
938 else printf("test_reordered_free_per_lcore() passed\n");
940 /*----------------------------*/
941 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
942 rte_eal_remote_launch(test_random_alloc_free, NULL, lcore_id);
945 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
946 if (rte_eal_wait_lcore(lcore_id) < 0)
950 printf("test_random_alloc_free() failed\n");
953 else printf("test_random_alloc_free() passed\n");
955 /*----------------------------*/
956 ret = test_rte_malloc_type_limits();
958 printf("test_rte_malloc_type_limits() failed\n");
961 /* TODO: uncomment following line once type limits are valid */
962 /*else printf("test_rte_malloc_type_limits() passed\n");*/
964 /*----------------------------*/
965 ret = test_rte_malloc_validate();
967 printf("test_rte_malloc_validate() failed\n");
970 else printf("test_rte_malloc_validate() passed\n");
972 ret = test_alloc_socket();
974 printf("test_alloc_socket() failed\n");
977 else printf("test_alloc_socket() passed\n");
979 ret = test_multi_alloc_statistics();
981 printf("test_multi_alloc_statistics() failed\n");
985 printf("test_multi_alloc_statistics() passed\n");
990 REGISTER_TEST_COMMAND(malloc_autotest, test_malloc);