4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 #include <sys/queue.h>
42 #include <rte_common.h>
43 #include <rte_memory.h>
44 #include <rte_memzone.h>
45 #include <rte_per_lcore.h>
46 #include <rte_launch.h>
48 #include <rte_per_lcore.h>
49 #include <rte_lcore.h>
50 #include <rte_malloc.h>
51 #include <rte_cycles.h>
52 #include <rte_random.h>
53 #include <rte_string_fns.h>
63 * Allocate some dynamic memory from heap (3 areas). Check that areas
64 * don't overlap and that alignment constraints match. This test is
65 * done many times on different lcores simultaneously.
68 /* Test if memory overlaps: return 1 if true, or 0 if false. */
70 is_memory_overlap(void *p1, size_t len1, void *p2, size_t len2)
72 unsigned long ptr1 = (unsigned long)p1;
73 unsigned long ptr2 = (unsigned long)p2;
75 if (ptr2 >= ptr1 && (ptr2 - ptr1) < len1)
77 else if (ptr2 < ptr1 && (ptr1 - ptr2) < len2)
83 is_aligned(void *p, int align)
85 unsigned long addr = (unsigned long)p;
86 unsigned mask = align - 1;
94 test_align_overlap_per_lcore(__attribute__((unused)) void *arg)
96 const unsigned align1 = 8,
100 void *p1 = NULL, *p2 = NULL, *p3 = NULL;
103 for (i = 0; i < N; i++) {
104 p1 = rte_zmalloc("dummy", 1000, align1);
106 printf("rte_zmalloc returned NULL (i=%u)\n", i);
110 for(j = 0; j < 1000 ; j++) {
111 if( *(char *)p1 != 0) {
112 printf("rte_zmalloc didn't zero"
113 "the allocated memory\n");
117 p2 = rte_malloc("dummy", 1000, align2);
119 printf("rte_malloc returned NULL (i=%u)\n", i);
124 p3 = rte_malloc("dummy", 1000, align3);
126 printf("rte_malloc returned NULL (i=%u)\n", i);
132 if (is_memory_overlap(p1, 1000, p2, 1000)) {
133 printf("p1 and p2 overlaps\n");
136 if (is_memory_overlap(p2, 1000, p3, 1000)) {
137 printf("p2 and p3 overlaps\n");
140 if (is_memory_overlap(p1, 1000, p3, 1000)) {
141 printf("p1 and p3 overlaps\n");
144 if (!is_aligned(p1, align1)) {
145 printf("p1 is not aligned\n");
148 if (!is_aligned(p2, align2)) {
149 printf("p2 is not aligned\n");
152 if (!is_aligned(p3, align3)) {
153 printf("p3 is not aligned\n");
160 rte_malloc_dump_stats(stdout, "dummy");
166 test_reordered_free_per_lcore(__attribute__((unused)) void *arg)
168 const unsigned align1 = 8,
175 for (i = 0; i < 30; i++) {
176 p1 = rte_zmalloc("dummy", 1000, align1);
178 printf("rte_zmalloc returned NULL (i=%u)\n", i);
182 for(j = 0; j < 1000 ; j++) {
183 if( *(char *)p1 != 0) {
184 printf("rte_zmalloc didn't zero"
185 "the allocated memory\n");
189 /* use calloc to allocate 1000 16-byte items this time */
190 p2 = rte_calloc("dummy", 1000, 16, align2);
191 /* for third request use regular malloc again */
192 p3 = rte_malloc("dummy", 1000, align3);
194 printf("rte_malloc returned NULL (i=%u)\n", i);
198 if (is_memory_overlap(p1, 1000, p2, 1000)) {
199 printf("p1 and p2 overlaps\n");
202 if (is_memory_overlap(p2, 1000, p3, 1000)) {
203 printf("p2 and p3 overlaps\n");
206 if (is_memory_overlap(p1, 1000, p3, 1000)) {
207 printf("p1 and p3 overlaps\n");
210 if (!is_aligned(p1, align1)) {
211 printf("p1 is not aligned\n");
214 if (!is_aligned(p2, align2)) {
215 printf("p2 is not aligned\n");
218 if (!is_aligned(p3, align3)) {
219 printf("p3 is not aligned\n");
222 /* try freeing in every possible order */
256 rte_malloc_dump_stats(stdout, "dummy");
261 /* test function inside the malloc lib*/
263 test_str_to_size(void)
269 {{ "5G", (uint64_t)5 * 1024 * 1024 *1024 },
270 {"0x20g", (uint64_t)0x20 * 1024 * 1024 *1024},
271 {"10M", 10 * 1024 * 1024},
272 {"050m", 050 * 1024 * 1024},
278 {"-1", 0}, /* negative values return 0 */
281 {"18446744073709551616", 0} /* ULLONG_MAX + 1 == out of range*/
284 for (i = 0; i < sizeof(test_values)/sizeof(test_values[0]); i++)
285 if (rte_str_to_size(test_values[i].str) != test_values[i].value)
291 test_multi_alloc_statistics(void)
294 struct rte_malloc_socket_stats pre_stats, post_stats ,first_stats, second_stats;
297 #ifndef RTE_LIBRTE_MALLOC_DEBUG
298 int trailer_size = 0;
300 int trailer_size = RTE_CACHE_LINE_SIZE;
302 int overhead = RTE_CACHE_LINE_SIZE + trailer_size;
304 rte_malloc_get_socket_stats(socket, &pre_stats);
306 void *p1 = rte_malloc_socket("stats", size , align, socket);
310 rte_malloc_dump_stats(stdout, "stats");
312 rte_malloc_get_socket_stats(socket,&post_stats);
313 /* Check statistics reported are correct */
314 /* All post stats should be equal to pre stats after alloc freed */
315 if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) &&
316 (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) &&
317 (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&&
318 (post_stats.alloc_count!=pre_stats.alloc_count)&&
319 (post_stats.free_count!=pre_stats.free_count)) {
320 printf("Malloc statistics are incorrect - freed alloc\n");
323 /* Check two consecutive allocations */
326 rte_malloc_get_socket_stats(socket,&pre_stats);
327 void *p2 = rte_malloc_socket("add", size ,align, socket);
330 rte_malloc_get_socket_stats(socket,&first_stats);
332 void *p3 = rte_malloc_socket("add2", size,align, socket);
336 rte_malloc_get_socket_stats(socket,&second_stats);
341 /* After freeing both allocations check stats return to original */
342 rte_malloc_get_socket_stats(socket, &post_stats);
344 if(second_stats.heap_totalsz_bytes != first_stats.heap_totalsz_bytes) {
345 printf("Incorrect heap statistics: Total size \n");
348 /* Check allocated size is equal to two additions plus overhead */
349 if(second_stats.heap_allocsz_bytes !=
350 size + overhead + first_stats.heap_allocsz_bytes) {
351 printf("Incorrect heap statistics: Allocated size \n");
354 /* Check that allocation count increments correctly i.e. +1 */
355 if (second_stats.alloc_count != first_stats.alloc_count + 1) {
356 printf("Incorrect heap statistics: Allocated count \n");
360 if (second_stats.free_count != first_stats.free_count){
361 printf("Incorrect heap statistics: Free count \n");
365 /* Make sure that we didn't touch our greatest chunk: 2 * 11M) */
366 if (post_stats.greatest_free_size != pre_stats.greatest_free_size) {
367 printf("Incorrect heap statistics: Greatest free size \n");
370 /* Free size must equal the original free size minus the new allocation*/
371 if (first_stats.heap_freesz_bytes <= second_stats.heap_freesz_bytes) {
372 printf("Incorrect heap statistics: Free size \n");
376 if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) &&
377 (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) &&
378 (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&&
379 (post_stats.alloc_count!=pre_stats.alloc_count)&&
380 (post_stats.free_count!=pre_stats.free_count)) {
381 printf("Malloc statistics are incorrect - freed alloc\n");
388 test_rte_malloc_type_limits(void)
390 /* The type-limits functionality is not yet implemented,
391 * so always return 0 no matter what the retval.
393 const char *typename = "limit_test";
394 rte_malloc_set_limit(typename, 64 * 1024);
395 rte_malloc_dump_stats(stdout, typename);
402 const char hello_str[] = "Hello, world!";
403 const unsigned size1 = 1024;
404 const unsigned size2 = size1 + 1024;
405 const unsigned size3 = size2;
406 const unsigned size4 = size3 + 1024;
408 /* test data is the same even if element is moved*/
409 char *ptr1 = rte_zmalloc(NULL, size1, RTE_CACHE_LINE_SIZE);
411 printf("NULL pointer returned from rte_zmalloc\n");
414 snprintf(ptr1, size1, "%s" ,hello_str);
415 char *ptr2 = rte_realloc(ptr1, size2, RTE_CACHE_LINE_SIZE);
418 printf("NULL pointer returned from rte_realloc\n");
422 printf("unexpected - ptr1 == ptr2\n");
424 if (strcmp(ptr2, hello_str) != 0){
425 printf("Error - lost data from pointed area\n");
430 for (i = strnlen(hello_str, sizeof(hello_str)); i < size1; i++)
432 printf("Bad data in realloc\n");
436 /* now allocate third element, free the second
437 * and resize third. It should not move. (ptr1 is now invalid)
439 char *ptr3 = rte_zmalloc(NULL, size3, RTE_CACHE_LINE_SIZE);
441 printf("NULL pointer returned from rte_zmalloc\n");
445 for (i = 0; i < size3; i++)
447 printf("Bad data in zmalloc\n");
453 /* first resize to half the size of the freed block */
454 char *ptr4 = rte_realloc(ptr3, size4, RTE_CACHE_LINE_SIZE);
456 printf("NULL pointer returned from rte_realloc\n");
461 printf("Unexpected - ptr4 != ptr3\n");
465 /* now resize again to the full size of the freed block */
466 ptr4 = rte_realloc(ptr3, size3 + size2 + size1, RTE_CACHE_LINE_SIZE);
468 printf("Unexpected - ptr4 != ptr3 on second resize\n");
474 /* now try a resize to a smaller size, see if it works */
475 const unsigned size5 = 1024;
476 const unsigned size6 = size5 / 2;
477 char *ptr5 = rte_malloc(NULL, size5, RTE_CACHE_LINE_SIZE);
479 printf("NULL pointer returned from rte_malloc\n");
482 char *ptr6 = rte_realloc(ptr5, size6, RTE_CACHE_LINE_SIZE);
484 printf("NULL pointer returned from rte_realloc\n");
489 printf("Error, resizing to a smaller size moved data\n");
495 /* check for behaviour changing alignment */
496 const unsigned size7 = 1024;
497 const unsigned orig_align = RTE_CACHE_LINE_SIZE;
498 unsigned new_align = RTE_CACHE_LINE_SIZE * 2;
499 char *ptr7 = rte_malloc(NULL, size7, orig_align);
501 printf("NULL pointer returned from rte_malloc\n");
504 /* calc an alignment we don't already have */
505 while(RTE_PTR_ALIGN(ptr7, new_align) == ptr7)
507 char *ptr8 = rte_realloc(ptr7, size7, new_align);
509 printf("NULL pointer returned from rte_realloc\n");
513 if (RTE_PTR_ALIGN(ptr8, new_align) != ptr8){
514 printf("Failure to re-align data\n");
520 /* test behaviour when there is a free block after current one,
521 * but its not big enough
523 unsigned size9 = 1024, size10 = 1024;
524 unsigned size11 = size9 + size10 + 256;
525 char *ptr9 = rte_malloc(NULL, size9, RTE_CACHE_LINE_SIZE);
527 printf("NULL pointer returned from rte_malloc\n");
530 char *ptr10 = rte_malloc(NULL, size10, RTE_CACHE_LINE_SIZE);
532 printf("NULL pointer returned from rte_malloc\n");
536 char *ptr11 = rte_realloc(ptr10, size11, RTE_CACHE_LINE_SIZE);
538 printf("NULL pointer returned from rte_realloc\n");
543 printf("Error, unexpected that realloc has not created new buffer\n");
549 /* check we don't crash if we pass null to realloc
550 * We should get a malloc of the size requested*/
551 const size_t size12 = 1024;
553 char *ptr12 = rte_realloc(NULL, size12, RTE_CACHE_LINE_SIZE);
555 printf("NULL pointer returned from rte_realloc\n");
558 if (rte_malloc_validate(ptr12, &size12_check) < 0 ||
559 size12_check != size12){
568 test_random_alloc_free(void *_ __attribute__((unused)))
571 struct mem_list *next;
577 rte_srand((unsigned)rte_rdtsc());
579 for (i = 0; i < N; i++){
580 unsigned free_mem = 0;
581 size_t allocated_size;
583 const unsigned mem_size = sizeof(struct mem_list) + \
584 rte_rand() % (64 * 1024);
585 const unsigned align = 1 << (rte_rand() % 12); /* up to 4k alignment */
586 struct mem_list *entry = rte_malloc(NULL,
590 if (RTE_PTR_ALIGN(entry, align)!= entry)
592 if (rte_malloc_validate(entry, &allocated_size) == -1
593 || allocated_size < mem_size)
595 memset(entry->data, rte_lcore_id(),
596 mem_size - sizeof(*entry));
597 entry->next = list_head;
598 if (rte_malloc_validate(entry, NULL) == -1)
603 /* switch to freeing the memory with a 20% probability */
604 free_mem = ((rte_rand() % 10) >= 8);
607 struct mem_list *entry = list_head;
608 list_head = list_head->next;
612 printf("Lcore %u allocated/freed %u blocks\n", rte_lcore_id(), count);
616 #define err_return() do { \
617 printf("%s: %d - Error\n", __func__, __LINE__); \
622 test_rte_malloc_validate(void)
624 const size_t request_size = 1024;
625 size_t allocated_size;
626 char *data_ptr = rte_malloc(NULL, request_size, RTE_CACHE_LINE_SIZE);
627 #ifdef RTE_LIBRTE_MALLOC_DEBUG
629 char *over_write_vals = NULL;
632 if (data_ptr == NULL) {
633 printf("%s: %d - Allocation error\n", __func__, __LINE__);
637 /* check that a null input returns -1 */
638 if (rte_malloc_validate(NULL, NULL) != -1)
641 /* check that we get ok on a valid pointer */
642 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
645 /* check that the returned size is ok */
646 if (allocated_size < request_size)
649 #ifdef RTE_LIBRTE_MALLOC_DEBUG
651 /****** change the header to be bad */
653 over_write_vals = (char *)((uintptr_t)data_ptr - sizeof(save_buf));
654 /* first save the data as a backup before overwriting it */
655 memcpy(save_buf, over_write_vals, sizeof(save_buf));
656 memset(over_write_vals, 1, sizeof(save_buf));
657 /* then run validate */
658 retval = rte_malloc_validate(data_ptr, NULL);
659 /* finally restore the data again */
660 memcpy(over_write_vals, save_buf, sizeof(save_buf));
661 /* check we previously had an error */
665 /* check all ok again */
666 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
669 /**** change the trailer to be bad */
670 over_write_vals = (char *)((uintptr_t)data_ptr + allocated_size);
671 /* first save the data as a backup before overwriting it */
672 memcpy(save_buf, over_write_vals, sizeof(save_buf));
673 memset(over_write_vals, 1, sizeof(save_buf));
674 /* then run validate */
675 retval = rte_malloc_validate(data_ptr, NULL);
676 /* finally restore the data again */
677 memcpy(over_write_vals, save_buf, sizeof(save_buf));
681 /* check all ok again */
682 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
696 test_zero_aligned_alloc(void)
698 char *p1 = rte_malloc(NULL,1024, 0);
701 if (!rte_is_aligned(p1, RTE_CACHE_LINE_SIZE))
708 if (p1) rte_free(p1);
713 test_malloc_bad_params(void)
715 const char *type = NULL;
717 unsigned align = RTE_CACHE_LINE_SIZE;
719 /* rte_malloc expected to return null with inappropriate size */
720 char *bad_ptr = rte_malloc(type, size, align);
724 /* rte_malloc expected to return null with inappropriate alignment */
728 bad_ptr = rte_malloc(type, size, align);
735 /* clean up pointer */
741 /* Check if memory is avilable on a specific socket */
743 is_mem_on_socket(int32_t socket)
745 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
748 for (i = 0; i < RTE_MAX_MEMSEG; i++) {
749 if (socket == ms[i].socket_id)
756 * Find what socket a memory address is on. Only works for addresses within
757 * memsegs, not heap or stack...
760 addr_to_socket(void * addr)
762 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
765 for (i = 0; i < RTE_MAX_MEMSEG; i++) {
766 if ((ms[i].addr <= addr) &&
768 ((uintptr_t)ms[i].addr + (uintptr_t)ms[i].len)))
769 return ms[i].socket_id;
774 /* Test using rte_[c|m|zm]alloc_socket() on a specific socket */
776 test_alloc_single_socket(int32_t socket)
778 const char *type = NULL;
779 const size_t size = 10;
780 const unsigned align = 0;
782 int32_t desired_socket = (socket == SOCKET_ID_ANY) ?
783 (int32_t)rte_socket_id() : socket;
785 /* Test rte_calloc_socket() */
786 mem = rte_calloc_socket(type, size, sizeof(char), align, socket);
789 if (addr_to_socket(mem) != desired_socket) {
795 /* Test rte_malloc_socket() */
796 mem = rte_malloc_socket(type, size, align, socket);
799 if (addr_to_socket(mem) != desired_socket) {
804 /* Test rte_zmalloc_socket() */
805 mem = rte_zmalloc_socket(type, size, align, socket);
808 if (addr_to_socket(mem) != desired_socket) {
818 test_alloc_socket(void)
820 unsigned socket_count = 0;
823 if (test_alloc_single_socket(SOCKET_ID_ANY) < 0)
826 for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
827 if (is_mem_on_socket(i)) {
829 if (test_alloc_single_socket(i) < 0) {
830 printf("Fail: rte_malloc_socket(..., %u) did not succeed\n",
836 if (test_alloc_single_socket(i) == 0) {
837 printf("Fail: rte_malloc_socket(..., %u) succeeded\n",
844 /* Print warnign if only a single socket, but don't fail the test */
845 if (socket_count < 2) {
846 printf("WARNING: alloc_socket test needs memory on multiple sockets!\n");
858 if (test_str_to_size() < 0){
859 printf("test_str_to_size() failed\n");
862 else printf("test_str_to_size() passed\n");
864 if (test_zero_aligned_alloc() < 0){
865 printf("test_zero_aligned_alloc() failed\n");
868 else printf("test_zero_aligned_alloc() passed\n");
870 if (test_malloc_bad_params() < 0){
871 printf("test_malloc_bad_params() failed\n");
874 else printf("test_malloc_bad_params() passed\n");
876 if (test_realloc() < 0){
877 printf("test_realloc() failed\n");
880 else printf("test_realloc() passed\n");
882 /*----------------------------*/
883 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
884 rte_eal_remote_launch(test_align_overlap_per_lcore, NULL, lcore_id);
887 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
888 if (rte_eal_wait_lcore(lcore_id) < 0)
892 printf("test_align_overlap_per_lcore() failed\n");
895 else printf("test_align_overlap_per_lcore() passed\n");
897 /*----------------------------*/
898 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
899 rte_eal_remote_launch(test_reordered_free_per_lcore, NULL, lcore_id);
902 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
903 if (rte_eal_wait_lcore(lcore_id) < 0)
907 printf("test_reordered_free_per_lcore() failed\n");
910 else printf("test_reordered_free_per_lcore() passed\n");
912 /*----------------------------*/
913 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
914 rte_eal_remote_launch(test_random_alloc_free, NULL, lcore_id);
917 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
918 if (rte_eal_wait_lcore(lcore_id) < 0)
922 printf("test_random_alloc_free() failed\n");
925 else printf("test_random_alloc_free() passed\n");
927 /*----------------------------*/
928 ret = test_rte_malloc_type_limits();
930 printf("test_rte_malloc_type_limits() failed\n");
933 /* TODO: uncomment following line once type limits are valid */
934 /*else printf("test_rte_malloc_type_limits() passed\n");*/
936 /*----------------------------*/
937 ret = test_rte_malloc_validate();
939 printf("test_rte_malloc_validate() failed\n");
942 else printf("test_rte_malloc_validate() passed\n");
944 ret = test_alloc_socket();
946 printf("test_alloc_socket() failed\n");
949 else printf("test_alloc_socket() passed\n");
951 ret = test_multi_alloc_statistics();
953 printf("test_multi_alloc_statistics() failed\n");
957 printf("test_multi_alloc_statistics() passed\n");
962 REGISTER_TEST_COMMAND(malloc_autotest, test_malloc);