4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 #include <sys/queue.h>
42 #include <rte_common.h>
43 #include <rte_memory.h>
44 #include <rte_per_lcore.h>
45 #include <rte_launch.h>
47 #include <rte_lcore.h>
48 #include <rte_malloc.h>
49 #include <rte_cycles.h>
50 #include <rte_random.h>
51 #include <rte_string_fns.h>
61 * Allocate some dynamic memory from heap (3 areas). Check that areas
62 * don't overlap and that alignment constraints match. This test is
63 * done many times on different lcores simultaneously.
66 /* Test if memory overlaps: return 1 if true, or 0 if false. */
68 is_memory_overlap(void *p1, size_t len1, void *p2, size_t len2)
70 unsigned long ptr1 = (unsigned long)p1;
71 unsigned long ptr2 = (unsigned long)p2;
73 if (ptr2 >= ptr1 && (ptr2 - ptr1) < len1)
75 else if (ptr2 < ptr1 && (ptr1 - ptr2) < len2)
81 is_aligned(void *p, int align)
83 unsigned long addr = (unsigned long)p;
84 unsigned mask = align - 1;
92 test_align_overlap_per_lcore(__attribute__((unused)) void *arg)
94 const unsigned align1 = 8,
98 void *p1 = NULL, *p2 = NULL, *p3 = NULL;
101 for (i = 0; i < N; i++) {
102 p1 = rte_zmalloc("dummy", 1000, align1);
104 printf("rte_zmalloc returned NULL (i=%u)\n", i);
108 for(j = 0; j < 1000 ; j++) {
109 if( *(char *)p1 != 0) {
110 printf("rte_zmalloc didn't zero the allocated memory\n");
114 p2 = rte_malloc("dummy", 1000, align2);
116 printf("rte_malloc returned NULL (i=%u)\n", i);
121 p3 = rte_malloc("dummy", 1000, align3);
123 printf("rte_malloc returned NULL (i=%u)\n", i);
129 if (is_memory_overlap(p1, 1000, p2, 1000)) {
130 printf("p1 and p2 overlaps\n");
133 if (is_memory_overlap(p2, 1000, p3, 1000)) {
134 printf("p2 and p3 overlaps\n");
137 if (is_memory_overlap(p1, 1000, p3, 1000)) {
138 printf("p1 and p3 overlaps\n");
141 if (!is_aligned(p1, align1)) {
142 printf("p1 is not aligned\n");
145 if (!is_aligned(p2, align2)) {
146 printf("p2 is not aligned\n");
149 if (!is_aligned(p3, align3)) {
150 printf("p3 is not aligned\n");
157 rte_malloc_dump_stats(stdout, "dummy");
163 test_reordered_free_per_lcore(__attribute__((unused)) void *arg)
165 const unsigned align1 = 8,
172 for (i = 0; i < 30; i++) {
173 p1 = rte_zmalloc("dummy", 1000, align1);
175 printf("rte_zmalloc returned NULL (i=%u)\n", i);
179 for(j = 0; j < 1000 ; j++) {
180 if( *(char *)p1 != 0) {
181 printf("rte_zmalloc didn't zero the allocated memory\n");
185 /* use calloc to allocate 1000 16-byte items this time */
186 p2 = rte_calloc("dummy", 1000, 16, align2);
187 /* for third request use regular malloc again */
188 p3 = rte_malloc("dummy", 1000, align3);
190 printf("rte_malloc returned NULL (i=%u)\n", i);
194 if (is_memory_overlap(p1, 1000, p2, 1000)) {
195 printf("p1 and p2 overlaps\n");
198 if (is_memory_overlap(p2, 1000, p3, 1000)) {
199 printf("p2 and p3 overlaps\n");
202 if (is_memory_overlap(p1, 1000, p3, 1000)) {
203 printf("p1 and p3 overlaps\n");
206 if (!is_aligned(p1, align1)) {
207 printf("p1 is not aligned\n");
210 if (!is_aligned(p2, align2)) {
211 printf("p2 is not aligned\n");
214 if (!is_aligned(p3, align3)) {
215 printf("p3 is not aligned\n");
218 /* try freeing in every possible order */
252 rte_malloc_dump_stats(stdout, "dummy");
257 /* test function inside the malloc lib*/
259 test_str_to_size(void)
265 {{ "5G", (uint64_t)5 * 1024 * 1024 *1024 },
266 {"0x20g", (uint64_t)0x20 * 1024 * 1024 *1024},
267 {"10M", 10 * 1024 * 1024},
268 {"050m", 050 * 1024 * 1024},
274 {"-1", 0}, /* negative values return 0 */
277 {"18446744073709551616", 0} /* ULLONG_MAX + 1 == out of range*/
280 for (i = 0; i < sizeof(test_values)/sizeof(test_values[0]); i++)
281 if (rte_str_to_size(test_values[i].str) != test_values[i].value)
287 test_multi_alloc_statistics(void)
290 struct rte_malloc_socket_stats pre_stats, post_stats ,first_stats, second_stats;
293 #ifndef RTE_MALLOC_DEBUG
294 int trailer_size = 0;
296 int trailer_size = RTE_CACHE_LINE_SIZE;
298 int overhead = RTE_CACHE_LINE_SIZE + trailer_size;
300 rte_malloc_get_socket_stats(socket, &pre_stats);
302 void *p1 = rte_malloc_socket("stats", size , align, socket);
306 rte_malloc_dump_stats(stdout, "stats");
308 rte_malloc_get_socket_stats(socket,&post_stats);
309 /* Check statistics reported are correct */
310 /* All post stats should be equal to pre stats after alloc freed */
311 if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) &&
312 (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) &&
313 (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&&
314 (post_stats.alloc_count!=pre_stats.alloc_count)&&
315 (post_stats.free_count!=pre_stats.free_count)) {
316 printf("Malloc statistics are incorrect - freed alloc\n");
319 /* Check two consecutive allocations */
322 rte_malloc_get_socket_stats(socket,&pre_stats);
323 void *p2 = rte_malloc_socket("add", size ,align, socket);
326 rte_malloc_get_socket_stats(socket,&first_stats);
328 void *p3 = rte_malloc_socket("add2", size,align, socket);
332 rte_malloc_get_socket_stats(socket,&second_stats);
337 /* After freeing both allocations check stats return to original */
338 rte_malloc_get_socket_stats(socket, &post_stats);
340 if(second_stats.heap_totalsz_bytes != first_stats.heap_totalsz_bytes) {
341 printf("Incorrect heap statistics: Total size \n");
344 /* Check allocated size is equal to two additions plus overhead */
345 if(second_stats.heap_allocsz_bytes !=
346 size + overhead + first_stats.heap_allocsz_bytes) {
347 printf("Incorrect heap statistics: Allocated size \n");
350 /* Check that allocation count increments correctly i.e. +1 */
351 if (second_stats.alloc_count != first_stats.alloc_count + 1) {
352 printf("Incorrect heap statistics: Allocated count \n");
356 if (second_stats.free_count != first_stats.free_count){
357 printf("Incorrect heap statistics: Free count \n");
361 /* Make sure that we didn't touch our greatest chunk: 2 * 11M) */
362 if (post_stats.greatest_free_size != pre_stats.greatest_free_size) {
363 printf("Incorrect heap statistics: Greatest free size \n");
366 /* Free size must equal the original free size minus the new allocation*/
367 if (first_stats.heap_freesz_bytes <= second_stats.heap_freesz_bytes) {
368 printf("Incorrect heap statistics: Free size \n");
372 if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) &&
373 (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) &&
374 (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&&
375 (post_stats.alloc_count!=pre_stats.alloc_count)&&
376 (post_stats.free_count!=pre_stats.free_count)) {
377 printf("Malloc statistics are incorrect - freed alloc\n");
384 test_rte_malloc_type_limits(void)
386 /* The type-limits functionality is not yet implemented,
387 * so always return 0 no matter what the retval.
389 const char *typename = "limit_test";
390 rte_malloc_set_limit(typename, 64 * 1024);
391 rte_malloc_dump_stats(stdout, typename);
398 const char hello_str[] = "Hello, world!";
399 const unsigned size1 = 1024;
400 const unsigned size2 = size1 + 1024;
401 const unsigned size3 = size2;
402 const unsigned size4 = size3 + 1024;
404 /* test data is the same even if element is moved*/
405 char *ptr1 = rte_zmalloc(NULL, size1, RTE_CACHE_LINE_SIZE);
407 printf("NULL pointer returned from rte_zmalloc\n");
410 snprintf(ptr1, size1, "%s" ,hello_str);
411 char *ptr2 = rte_realloc(ptr1, size2, RTE_CACHE_LINE_SIZE);
414 printf("NULL pointer returned from rte_realloc\n");
418 printf("unexpected - ptr1 == ptr2\n");
420 if (strcmp(ptr2, hello_str) != 0){
421 printf("Error - lost data from pointed area\n");
426 for (i = strnlen(hello_str, sizeof(hello_str)); i < size1; i++)
428 printf("Bad data in realloc\n");
432 /* now allocate third element, free the second
433 * and resize third. It should not move. (ptr1 is now invalid)
435 char *ptr3 = rte_zmalloc(NULL, size3, RTE_CACHE_LINE_SIZE);
437 printf("NULL pointer returned from rte_zmalloc\n");
441 for (i = 0; i < size3; i++)
443 printf("Bad data in zmalloc\n");
449 /* first resize to half the size of the freed block */
450 char *ptr4 = rte_realloc(ptr3, size4, RTE_CACHE_LINE_SIZE);
452 printf("NULL pointer returned from rte_realloc\n");
457 printf("Unexpected - ptr4 != ptr3\n");
461 /* now resize again to the full size of the freed block */
462 ptr4 = rte_realloc(ptr3, size3 + size2 + size1, RTE_CACHE_LINE_SIZE);
464 printf("Unexpected - ptr4 != ptr3 on second resize\n");
470 /* now try a resize to a smaller size, see if it works */
471 const unsigned size5 = 1024;
472 const unsigned size6 = size5 / 2;
473 char *ptr5 = rte_malloc(NULL, size5, RTE_CACHE_LINE_SIZE);
475 printf("NULL pointer returned from rte_malloc\n");
478 char *ptr6 = rte_realloc(ptr5, size6, RTE_CACHE_LINE_SIZE);
480 printf("NULL pointer returned from rte_realloc\n");
485 printf("Error, resizing to a smaller size moved data\n");
491 /* check for behaviour changing alignment */
492 const unsigned size7 = 1024;
493 const unsigned orig_align = RTE_CACHE_LINE_SIZE;
494 unsigned new_align = RTE_CACHE_LINE_SIZE * 2;
495 char *ptr7 = rte_malloc(NULL, size7, orig_align);
497 printf("NULL pointer returned from rte_malloc\n");
500 /* calc an alignment we don't already have */
501 while(RTE_PTR_ALIGN(ptr7, new_align) == ptr7)
503 char *ptr8 = rte_realloc(ptr7, size7, new_align);
505 printf("NULL pointer returned from rte_realloc\n");
509 if (RTE_PTR_ALIGN(ptr8, new_align) != ptr8){
510 printf("Failure to re-align data\n");
516 /* test behaviour when there is a free block after current one,
517 * but its not big enough
519 unsigned size9 = 1024, size10 = 1024;
520 unsigned size11 = size9 + size10 + 256;
521 char *ptr9 = rte_malloc(NULL, size9, RTE_CACHE_LINE_SIZE);
523 printf("NULL pointer returned from rte_malloc\n");
526 char *ptr10 = rte_malloc(NULL, size10, RTE_CACHE_LINE_SIZE);
528 printf("NULL pointer returned from rte_malloc\n");
532 char *ptr11 = rte_realloc(ptr10, size11, RTE_CACHE_LINE_SIZE);
534 printf("NULL pointer returned from rte_realloc\n");
539 printf("Error, unexpected that realloc has not created new buffer\n");
545 /* check we don't crash if we pass null to realloc
546 * We should get a malloc of the size requested*/
547 const size_t size12 = 1024;
549 char *ptr12 = rte_realloc(NULL, size12, RTE_CACHE_LINE_SIZE);
551 printf("NULL pointer returned from rte_realloc\n");
554 if (rte_malloc_validate(ptr12, &size12_check) < 0 ||
555 size12_check != size12){
564 test_random_alloc_free(void *_ __attribute__((unused)))
567 struct mem_list *next;
573 rte_srand((unsigned)rte_rdtsc());
575 for (i = 0; i < N; i++){
576 unsigned free_mem = 0;
577 size_t allocated_size;
579 const unsigned mem_size = sizeof(struct mem_list) + \
580 rte_rand() % (64 * 1024);
581 const unsigned align = 1 << (rte_rand() % 12); /* up to 4k alignment */
582 struct mem_list *entry = rte_malloc(NULL,
586 if (RTE_PTR_ALIGN(entry, align)!= entry)
588 if (rte_malloc_validate(entry, &allocated_size) == -1
589 || allocated_size < mem_size)
591 memset(entry->data, rte_lcore_id(),
592 mem_size - sizeof(*entry));
593 entry->next = list_head;
594 if (rte_malloc_validate(entry, NULL) == -1)
599 /* switch to freeing the memory with a 20% probability */
600 free_mem = ((rte_rand() % 10) >= 8);
603 struct mem_list *entry = list_head;
604 list_head = list_head->next;
608 printf("Lcore %u allocated/freed %u blocks\n", rte_lcore_id(), count);
612 #define err_return() do { \
613 printf("%s: %d - Error\n", __func__, __LINE__); \
618 test_rte_malloc_validate(void)
620 const size_t request_size = 1024;
621 size_t allocated_size;
622 char *data_ptr = rte_malloc(NULL, request_size, RTE_CACHE_LINE_SIZE);
623 #ifdef RTE_MALLOC_DEBUG
625 char *over_write_vals = NULL;
628 if (data_ptr == NULL) {
629 printf("%s: %d - Allocation error\n", __func__, __LINE__);
633 /* check that a null input returns -1 */
634 if (rte_malloc_validate(NULL, NULL) != -1)
637 /* check that we get ok on a valid pointer */
638 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
641 /* check that the returned size is ok */
642 if (allocated_size < request_size)
645 #ifdef RTE_MALLOC_DEBUG
647 /****** change the header to be bad */
649 over_write_vals = (char *)((uintptr_t)data_ptr - sizeof(save_buf));
650 /* first save the data as a backup before overwriting it */
651 memcpy(save_buf, over_write_vals, sizeof(save_buf));
652 memset(over_write_vals, 1, sizeof(save_buf));
653 /* then run validate */
654 retval = rte_malloc_validate(data_ptr, NULL);
655 /* finally restore the data again */
656 memcpy(over_write_vals, save_buf, sizeof(save_buf));
657 /* check we previously had an error */
661 /* check all ok again */
662 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
665 /**** change the trailer to be bad */
666 over_write_vals = (char *)((uintptr_t)data_ptr + allocated_size);
667 /* first save the data as a backup before overwriting it */
668 memcpy(save_buf, over_write_vals, sizeof(save_buf));
669 memset(over_write_vals, 1, sizeof(save_buf));
670 /* then run validate */
671 retval = rte_malloc_validate(data_ptr, NULL);
672 /* finally restore the data again */
673 memcpy(over_write_vals, save_buf, sizeof(save_buf));
677 /* check all ok again */
678 if (rte_malloc_validate(data_ptr, &allocated_size) < 0)
692 test_zero_aligned_alloc(void)
694 char *p1 = rte_malloc(NULL,1024, 0);
697 if (!rte_is_aligned(p1, RTE_CACHE_LINE_SIZE))
704 if (p1) rte_free(p1);
709 test_malloc_bad_params(void)
711 const char *type = NULL;
713 unsigned align = RTE_CACHE_LINE_SIZE;
715 /* rte_malloc expected to return null with inappropriate size */
716 char *bad_ptr = rte_malloc(type, size, align);
720 /* rte_malloc expected to return null with inappropriate alignment */
724 bad_ptr = rte_malloc(type, size, align);
731 /* clean up pointer */
737 /* Check if memory is available on a specific socket */
739 is_mem_on_socket(int32_t socket)
741 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
744 for (i = 0; i < RTE_MAX_MEMSEG; i++) {
745 if (socket == ms[i].socket_id)
752 * Find what socket a memory address is on. Only works for addresses within
753 * memsegs, not heap or stack...
756 addr_to_socket(void * addr)
758 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
761 for (i = 0; i < RTE_MAX_MEMSEG; i++) {
762 if ((ms[i].addr <= addr) &&
764 ((uintptr_t)ms[i].addr + (uintptr_t)ms[i].len)))
765 return ms[i].socket_id;
770 /* Test using rte_[c|m|zm]alloc_socket() on a specific socket */
772 test_alloc_single_socket(int32_t socket)
774 const char *type = NULL;
775 const size_t size = 10;
776 const unsigned align = 0;
778 int32_t desired_socket = (socket == SOCKET_ID_ANY) ?
779 (int32_t)rte_socket_id() : socket;
781 /* Test rte_calloc_socket() */
782 mem = rte_calloc_socket(type, size, sizeof(char), align, socket);
785 if (addr_to_socket(mem) != desired_socket) {
791 /* Test rte_malloc_socket() */
792 mem = rte_malloc_socket(type, size, align, socket);
795 if (addr_to_socket(mem) != desired_socket) {
800 /* Test rte_zmalloc_socket() */
801 mem = rte_zmalloc_socket(type, size, align, socket);
804 if (addr_to_socket(mem) != desired_socket) {
814 test_alloc_socket(void)
816 unsigned socket_count = 0;
819 if (test_alloc_single_socket(SOCKET_ID_ANY) < 0)
822 for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
823 if (is_mem_on_socket(i)) {
825 if (test_alloc_single_socket(i) < 0) {
826 printf("Fail: rte_malloc_socket(..., %u) did not succeed\n",
832 if (test_alloc_single_socket(i) == 0) {
833 printf("Fail: rte_malloc_socket(..., %u) succeeded\n",
840 /* Print warnign if only a single socket, but don't fail the test */
841 if (socket_count < 2) {
842 printf("WARNING: alloc_socket test needs memory on multiple sockets!\n");
854 if (test_str_to_size() < 0){
855 printf("test_str_to_size() failed\n");
858 else printf("test_str_to_size() passed\n");
860 if (test_zero_aligned_alloc() < 0){
861 printf("test_zero_aligned_alloc() failed\n");
864 else printf("test_zero_aligned_alloc() passed\n");
866 if (test_malloc_bad_params() < 0){
867 printf("test_malloc_bad_params() failed\n");
870 else printf("test_malloc_bad_params() passed\n");
872 if (test_realloc() < 0){
873 printf("test_realloc() failed\n");
876 else printf("test_realloc() passed\n");
878 /*----------------------------*/
879 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
880 rte_eal_remote_launch(test_align_overlap_per_lcore, NULL, lcore_id);
883 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
884 if (rte_eal_wait_lcore(lcore_id) < 0)
888 printf("test_align_overlap_per_lcore() failed\n");
891 else printf("test_align_overlap_per_lcore() passed\n");
893 /*----------------------------*/
894 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
895 rte_eal_remote_launch(test_reordered_free_per_lcore, NULL, lcore_id);
898 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
899 if (rte_eal_wait_lcore(lcore_id) < 0)
903 printf("test_reordered_free_per_lcore() failed\n");
906 else printf("test_reordered_free_per_lcore() passed\n");
908 /*----------------------------*/
909 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
910 rte_eal_remote_launch(test_random_alloc_free, NULL, lcore_id);
913 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
914 if (rte_eal_wait_lcore(lcore_id) < 0)
918 printf("test_random_alloc_free() failed\n");
921 else printf("test_random_alloc_free() passed\n");
923 /*----------------------------*/
924 ret = test_rte_malloc_type_limits();
926 printf("test_rte_malloc_type_limits() failed\n");
929 /* TODO: uncomment following line once type limits are valid */
930 /*else printf("test_rte_malloc_type_limits() passed\n");*/
932 /*----------------------------*/
933 ret = test_rte_malloc_validate();
935 printf("test_rte_malloc_validate() failed\n");
938 else printf("test_rte_malloc_validate() passed\n");
940 ret = test_alloc_socket();
942 printf("test_alloc_socket() failed\n");
945 else printf("test_alloc_socket() passed\n");
947 ret = test_multi_alloc_statistics();
949 printf("test_multi_alloc_statistics() failed\n");
953 printf("test_multi_alloc_statistics() passed\n");
958 REGISTER_TEST_COMMAND(malloc_autotest, test_malloc);