4 * Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 #include <sys/queue.h>
44 #include <rte_common.h>
46 #include <rte_debug.h>
47 #include <rte_memory.h>
48 #include <rte_memzone.h>
49 #include <rte_launch.h>
50 #include <rte_cycles.h>
51 #include <rte_tailq.h>
53 #include <rte_per_lcore.h>
54 #include <rte_lcore.h>
55 #include <rte_atomic.h>
56 #include <rte_branch_prediction.h>
58 #include <rte_mempool.h>
59 #include <rte_spinlock.h>
60 #include <rte_malloc.h>
62 #include <cmdline_parse.h>
70 * #. Basic tests: done on one core with and without cache:
72 * - Get one object, put one object
73 * - Get two objects, put two objects
74 * - Get all objects, test that their content is not modified and
75 * put them back in the pool.
77 * #. Performance tests:
79 * Each core get *n_keep* objects per bulk of *n_get_bulk*. Then,
80 * objects are put back in the pool per bulk of *n_put_bulk*.
82 * This sequence is done during TIME_S seconds.
84 * This test is done on the following configurations:
86 * - Cores configuration (*cores*)
88 * - One core with cache
89 * - Two cores with cache
90 * - Max. cores with cache
91 * - One core without cache
92 * - Two cores without cache
93 * - Max. cores without cache
95 * - Bulk size (*n_get_bulk*, *n_put_bulk*)
97 * - Bulk get from 1 to 32
98 * - Bulk put from 1 to 32
100 * - Number of kept objects (*n_keep*)
108 #define MEMPOOL_ELT_SIZE 2048
110 #define MEMPOOL_SIZE ((RTE_MAX_LCORE*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1)
112 static struct rte_mempool *mp;
113 static struct rte_mempool *mp_cache, *mp_nocache;
115 static rte_atomic32_t synchro;
117 /* number of objects in one bulk operation (get or put) */
118 static unsigned n_get_bulk;
119 static unsigned n_put_bulk;
121 /* number of objects retrived from mempool before putting them back */
122 static unsigned n_keep;
124 /* number of enqueues / dequeues */
125 struct mempool_test_stats {
127 } __rte_cache_aligned;
129 static struct mempool_test_stats stats[RTE_MAX_LCORE];
132 per_lcore_mempool_test(__attribute__((unused)) void *arg)
134 void *obj_table[MAX_KEEP];
136 unsigned lcore_id = rte_lcore_id();
138 uint64_t start_cycles, end_cycles;
139 uint64_t time_diff = 0, hz = rte_get_hpet_hz();
141 /* n_get_bulk and n_put_bulk must be divisors of n_keep */
142 if (((n_keep / n_get_bulk) * n_get_bulk) != n_keep)
144 if (((n_keep / n_put_bulk) * n_put_bulk) != n_keep)
147 stats[lcore_id].enq_count = 0;
149 /* wait synchro for slaves */
150 if (lcore_id != rte_get_master_lcore())
151 while (rte_atomic32_read(&synchro) == 0);
153 start_cycles = rte_get_hpet_cycles();
155 while (time_diff/hz < TIME_S) {
156 for (i = 0; likely(i < (N/n_keep)); i++) {
157 /* get n_keep objects by bulk of n_bulk */
159 while (idx < n_keep) {
160 ret = rte_mempool_get_bulk(mp, &obj_table[idx],
162 if (unlikely(ret < 0)) {
163 rte_mempool_dump(mp);
164 rte_ring_dump(mp->ring);
165 /* in this case, objects are lost... */
171 /* put the objects back */
173 while (idx < n_keep) {
174 rte_mempool_put_bulk(mp, &obj_table[idx],
179 end_cycles = rte_get_hpet_cycles();
180 time_diff = end_cycles - start_cycles;
181 stats[lcore_id].enq_count += N;
187 /* launch all the per-lcore test, and display the result */
189 launch_cores(unsigned cores)
194 unsigned cores_save = cores;
196 rte_atomic32_set(&synchro, 0);
199 memset(stats, 0, sizeof(stats));
201 printf("mempool_autotest cache=%u cores=%u n_get_bulk=%u "
202 "n_put_bulk=%u n_keep=%u ",
203 (unsigned) mp->cache_size, cores, n_get_bulk, n_put_bulk, n_keep);
205 if (rte_mempool_count(mp) != MEMPOOL_SIZE) {
206 printf("mempool is not full\n");
210 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
214 rte_eal_remote_launch(per_lcore_mempool_test,
218 /* start synchro and launch test on master */
219 rte_atomic32_set(&synchro, 1);
221 ret = per_lcore_mempool_test(NULL);
224 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
228 if (rte_eal_wait_lcore(lcore_id) < 0)
233 printf("per-lcore test returned -1\n");
238 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++)
239 rate += (stats[lcore_id].enq_count / TIME_S);
241 printf("rate_persec=%u\n", rate);
246 /* for a given number of core, launch all test cases */
248 do_one_mempool_test(unsigned cores)
250 unsigned bulk_tab_get[] = { 1, 4, 32, 0 };
251 unsigned bulk_tab_put[] = { 1, 4, 32, 0 };
252 unsigned keep_tab[] = { 32, 128, 0 };
253 unsigned *get_bulk_ptr;
254 unsigned *put_bulk_ptr;
258 for (get_bulk_ptr = bulk_tab_get; *get_bulk_ptr; get_bulk_ptr++) {
259 for (put_bulk_ptr = bulk_tab_put; *put_bulk_ptr; put_bulk_ptr++) {
260 for (keep_ptr = keep_tab; *keep_ptr; keep_ptr++) {
262 n_get_bulk = *get_bulk_ptr;
263 n_put_bulk = *put_bulk_ptr;
265 ret = launch_cores(cores);
277 * save the object number in the first 4 bytes of object data. All
278 * other bytes are set to 0.
281 my_obj_init(struct rte_mempool *mp, __attribute__((unused)) void *arg,
282 void *obj, unsigned i)
284 uint32_t *objnum = obj;
285 memset(obj, 0, mp->elt_size);
289 /* basic tests (done on one core) */
291 test_mempool_basic(void)
299 unsigned old_bulk_count;
301 /* dump the mempool status */
302 rte_mempool_dump(mp);
303 old_bulk_count = rte_mempool_get_bulk_count(mp);
304 rte_mempool_dump(mp);
305 if (rte_mempool_set_bulk_count(mp, 0) == 0)
307 if (rte_mempool_get_bulk_count(mp) == 0)
309 if (rte_mempool_set_bulk_count(mp, 2) < 0)
311 if (rte_mempool_get_bulk_count(mp) != 2)
313 rte_mempool_dump(mp);
314 if (rte_mempool_set_bulk_count(mp, old_bulk_count) < 0)
316 if (rte_mempool_get_bulk_count(mp) != old_bulk_count)
318 rte_mempool_dump(mp);
320 printf("get an object\n");
321 if (rte_mempool_get(mp, &obj) < 0)
323 rte_mempool_dump(mp);
325 printf("put the object back\n");
326 rte_mempool_put(mp, obj);
327 rte_mempool_dump(mp);
329 printf("get 2 objects\n");
330 if (rte_mempool_get(mp, &obj) < 0)
332 if (rte_mempool_get(mp, &obj2) < 0) {
333 rte_mempool_put(mp, obj);
336 rte_mempool_dump(mp);
338 printf("put the objects back\n");
339 rte_mempool_put(mp, obj);
340 rte_mempool_put(mp, obj2);
341 rte_mempool_dump(mp);
344 * get many objects: we cannot get them all because the cache
345 * on other cores may not be empty.
347 objtable = malloc(MEMPOOL_SIZE * sizeof(void *));
348 if (objtable == NULL) {
352 for (i=0; i<MEMPOOL_SIZE; i++) {
353 if (rte_mempool_get(mp, &objtable[i]) < 0)
358 * for each object, check that its content was not modified,
359 * and put objects back in pool
365 if (*objnum > MEMPOOL_SIZE) {
366 printf("bad object number\n");
370 for (j=sizeof(*objnum); j<mp->elt_size; j++) {
371 if (obj_data[j] != 0)
375 rte_mempool_put(mp, objtable[i]);
380 printf("objects were modified!\n");
385 static int test_mempool_creation_with_exceeded_cache_size(void)
387 struct rte_mempool *mp_cov;
389 mp_cov = rte_mempool_create("test_mempool_creation_with_exceeded_cache_size", MEMPOOL_SIZE,
391 RTE_MEMPOOL_CACHE_MAX_SIZE + 32, 0,
402 static struct rte_mempool *mp_spsc;
403 static rte_spinlock_t scsp_spinlock;
404 static void *scsp_obj_table[MAX_KEEP];
407 * single producer function
409 static int test_mempool_single_producer(void)
413 uint64_t start_cycles, end_cycles;
414 uint64_t duration = rte_get_hpet_hz() * 8;
416 start_cycles = rte_get_hpet_cycles();
418 end_cycles = rte_get_hpet_cycles();
419 /* duration uses up, stop producing */
420 if (start_cycles + duration < end_cycles)
422 rte_spinlock_lock(&scsp_spinlock);
423 for (i = 0; i < MAX_KEEP; i ++) {
424 if (NULL != scsp_obj_table[i])
425 obj = scsp_obj_table[i];
428 rte_spinlock_unlock(&scsp_spinlock);
432 if (rte_mempool_from_obj(obj) != mp_spsc) {
433 printf("test_mempool_single_producer there is an obj not owned by this mempool\n");
436 rte_mempool_sp_put(mp_spsc, obj);
437 rte_spinlock_lock(&scsp_spinlock);
438 scsp_obj_table[i] = NULL;
439 rte_spinlock_unlock(&scsp_spinlock);
446 * single consumer function
448 static int test_mempool_single_consumer(void)
452 uint64_t start_cycles, end_cycles;
453 uint64_t duration = rte_get_hpet_hz() * 5;
455 start_cycles = rte_get_hpet_cycles();
457 end_cycles = rte_get_hpet_cycles();
458 /* duration uses up, stop consuming */
459 if (start_cycles + duration < end_cycles)
461 rte_spinlock_lock(&scsp_spinlock);
462 for (i = 0; i < MAX_KEEP; i ++) {
463 if (NULL == scsp_obj_table[i])
466 rte_spinlock_unlock(&scsp_spinlock);
469 if (rte_mempool_sc_get(mp_spsc, &obj) < 0)
471 rte_spinlock_lock(&scsp_spinlock);
472 scsp_obj_table[i] = obj;
473 rte_spinlock_unlock(&scsp_spinlock);
480 * test function for mempool test based on singple consumer and single producer, can run on one lcore only
482 static int test_mempool_launch_single_consumer(__attribute__((unused)) void *arg)
484 return test_mempool_single_consumer();
487 static void my_mp_init(struct rte_mempool * mp, __attribute__((unused)) void * arg)
489 printf("mempool name is %s\n", mp->name);
490 /* nothing to be implemented here*/
495 * it tests the mempool operations based on singple producer and single consumer
498 test_mempool_sp_sc(void)
501 unsigned lcore_id = rte_lcore_id();
504 /* create a mempool with single producer/consumer ring */
505 if (NULL == mp_spsc) {
506 mp_spsc = rte_mempool_create("test_mempool_sp_sc", MEMPOOL_SIZE,
507 MEMPOOL_ELT_SIZE, 0, 0,
510 SOCKET_ID_ANY, MEMPOOL_F_NO_CACHE_ALIGN | MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET);
511 if (NULL == mp_spsc) {
515 if (rte_mempool_lookup("test_mempool_sp_sc") != mp_spsc) {
516 printf("Cannot lookup mempool from its name\n");
519 lcore_next = rte_get_next_lcore(lcore_id, 0, 1);
520 if (RTE_MAX_LCORE <= lcore_next)
522 if (rte_eal_lcore_role(lcore_next) != ROLE_RTE)
524 rte_spinlock_init(&scsp_spinlock);
525 memset(scsp_obj_table, 0, sizeof(scsp_obj_table));
526 rte_eal_remote_launch(test_mempool_launch_single_consumer, NULL, lcore_next);
527 if(test_mempool_single_producer() < 0)
530 if(rte_eal_wait_lcore(lcore_next) < 0)
537 * it tests some more basic of mempool
540 test_mempool_basic_ex(struct rte_mempool * mp)
550 obj = (void **)rte_zmalloc("test_mempool_basic_ex", (MEMPOOL_SIZE * sizeof(void *)), 0);
552 printf("test_mempool_basic_ex fail to rte_malloc\n");
555 printf("test_mempool_basic_ex now mempool (%s) has %u free entries\n", mp->name, rte_mempool_free_count(mp));
556 if (rte_mempool_full(mp) != 1) {
557 printf("test_mempool_basic_ex the mempool is not full but it should be\n");
558 goto fail_mp_basic_ex;
561 for (i = 0; i < MEMPOOL_SIZE; i ++) {
562 if (rte_mempool_mc_get(mp, &obj[i]) < 0) {
563 printf("fail_mp_basic_ex fail to get mempool object for [%u]\n", i);
564 goto fail_mp_basic_ex;
567 if (rte_mempool_mc_get(mp, &err_obj) == 0) {
568 printf("test_mempool_basic_ex get an impossible obj from mempool\n");
569 goto fail_mp_basic_ex;
571 printf("number: %u\n", i);
572 if (rte_mempool_empty(mp) != 1) {
573 printf("test_mempool_basic_ex the mempool is not empty but it should be\n");
574 goto fail_mp_basic_ex;
577 for (i = 0; i < MEMPOOL_SIZE; i ++) {
578 rte_mempool_mp_put(mp, obj[i]);
580 if (rte_mempool_full(mp) != 1) {
581 printf("test_mempool_basic_ex the mempool is not full but it should be\n");
582 goto fail_mp_basic_ex;
589 rte_free((void *)obj);
595 test_mempool_same_name_twice_creation(void)
597 struct rte_mempool *mp_tc;
599 mp_tc = rte_mempool_create("test_mempool_same_name_twice_creation", MEMPOOL_SIZE,
600 MEMPOOL_ELT_SIZE, 0, 0,
607 mp_tc = rte_mempool_create("test_mempool_same_name_twice_creation", MEMPOOL_SIZE,
608 MEMPOOL_ELT_SIZE, 0, 0,
621 rte_atomic32_init(&synchro);
623 /* create a mempool (without cache) */
624 if (mp_nocache == NULL)
625 mp_nocache = rte_mempool_create("test_nocache", MEMPOOL_SIZE,
626 MEMPOOL_ELT_SIZE, 0, 0,
630 if (mp_nocache == NULL)
633 /* create a mempool (with cache) */
634 if (mp_cache == NULL)
635 mp_cache = rte_mempool_create("test_cache", MEMPOOL_SIZE,
637 RTE_MEMPOOL_CACHE_MAX_SIZE, 0,
641 if (mp_cache == NULL)
645 /* retrieve the mempool from its name */
646 if (rte_mempool_lookup("test_nocache") != mp_nocache) {
647 printf("Cannot lookup mempool from its name\n");
651 rte_mempool_list_dump();
653 /* basic tests without cache */
655 if (test_mempool_basic() < 0)
658 /* basic tests with cache */
660 if (test_mempool_basic() < 0)
663 /* more basic tests without cache */
664 if (test_mempool_basic_ex(mp_nocache) < 0)
667 /* performance test with 1, 2 and max cores */
668 printf("start performance test (without cache)\n");
671 if (do_one_mempool_test(1) < 0)
674 if (do_one_mempool_test(2) < 0)
677 if (do_one_mempool_test(rte_lcore_count()) < 0)
680 /* performance test with 1, 2 and max cores */
681 printf("start performance test (with cache)\n");
684 if (do_one_mempool_test(1) < 0)
687 if (do_one_mempool_test(2) < 0)
690 if (do_one_mempool_test(rte_lcore_count()) < 0)
693 /* mempool operation test based on single producer and single comsumer */
694 if (test_mempool_sp_sc() < 0)
697 if (test_mempool_creation_with_exceeded_cache_size() < 0)
700 if (test_mempool_same_name_twice_creation() < 0)
703 rte_mempool_list_dump();