1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2018 Arm Limited
9 #include <rte_rcu_qsbr.h>
11 #include <rte_hash_crc.h>
12 #include <rte_malloc.h>
13 #include <rte_cycles.h>
18 /* Check condition and return an error if true. */
19 static uint16_t enabled_core_ids[RTE_MAX_LCORE];
20 static uint8_t num_cores;
22 static uint32_t *keys;
23 #define TOTAL_ENTRY (1024 * 8)
24 #define COUNTER_VALUE 4096
25 static uint32_t *hash_data[TOTAL_ENTRY];
26 static volatile uint8_t writer_done;
27 static volatile uint8_t all_registered;
28 static volatile uint32_t thr_id;
30 static struct rte_rcu_qsbr *t[RTE_MAX_LCORE];
31 static struct rte_hash *h;
32 static char hash_name[8];
33 static rte_atomic64_t updates, checks;
34 static rte_atomic64_t update_cycles, check_cycles;
36 /* Scale down results to 1000 operations to support lower
39 #define RCU_SCALE_DOWN 1000
41 /* Simple way to allocate thread ids in 0 to RTE_MAX_LCORE space */
42 static inline uint32_t
47 tmp_thr_id = __atomic_fetch_add(&thr_id, 1, __ATOMIC_RELAXED);
48 if (tmp_thr_id >= RTE_MAX_LCORE)
49 printf("Invalid thread id %u\n", tmp_thr_id);
55 test_rcu_qsbr_reader_perf(void *arg)
57 bool writer_present = (bool)arg;
58 uint32_t thread_id = alloc_thread_id();
59 uint64_t loop_cnt = 0;
60 uint64_t begin, cycles;
62 /* Register for report QS */
63 rte_rcu_qsbr_thread_register(t[0], thread_id);
64 /* Make the thread online */
65 rte_rcu_qsbr_thread_online(t[0], thread_id);
67 begin = rte_rdtsc_precise();
70 while (!writer_done) {
71 /* Update quiescent state counter */
72 rte_rcu_qsbr_quiescent(t[0], thread_id);
76 while (loop_cnt < 100000000) {
77 /* Update quiescent state counter */
78 rte_rcu_qsbr_quiescent(t[0], thread_id);
83 cycles = rte_rdtsc_precise() - begin;
84 rte_atomic64_add(&update_cycles, cycles);
85 rte_atomic64_add(&updates, loop_cnt);
87 /* Make the thread offline */
88 rte_rcu_qsbr_thread_offline(t[0], thread_id);
89 /* Unregister before exiting to avoid writer from waiting */
90 rte_rcu_qsbr_thread_unregister(t[0], thread_id);
96 test_rcu_qsbr_writer_perf(void *arg)
98 bool wait = (bool)arg;
100 uint64_t loop_cnt = 0;
101 uint64_t begin, cycles;
103 begin = rte_rdtsc_precise();
106 /* Start the quiescent state query process */
108 token = rte_rcu_qsbr_start(t[0]);
110 /* Check quiescent state status */
111 rte_rcu_qsbr_check(t[0], token, wait);
113 } while (loop_cnt < 20000000);
115 cycles = rte_rdtsc_precise() - begin;
116 rte_atomic64_add(&check_cycles, cycles);
117 rte_atomic64_add(&checks, loop_cnt);
122 * Perf test: Reader/writer
123 * Single writer, Multiple Readers, Single QS var, Non-Blocking rcu_qsbr_check
126 test_rcu_qsbr_perf(void)
133 rte_atomic64_clear(&updates);
134 rte_atomic64_clear(&update_cycles);
135 rte_atomic64_clear(&checks);
136 rte_atomic64_clear(&check_cycles);
138 printf("\nPerf Test: %d Readers/1 Writer('wait' in qsbr_check == true)\n",
141 __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
143 if (all_registered == 1)
144 tmp_num_cores = num_cores - 1;
146 tmp_num_cores = RTE_MAX_LCORE;
148 sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
149 t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
150 RTE_CACHE_LINE_SIZE);
151 /* QS variable is initialized */
152 rte_rcu_qsbr_init(t[0], tmp_num_cores);
154 /* Reader threads are launched */
155 for (i = 0; i < num_cores - 1; i++)
156 rte_eal_remote_launch(test_rcu_qsbr_reader_perf, (void *)1,
157 enabled_core_ids[i]);
159 /* Writer thread is launched */
160 rte_eal_remote_launch(test_rcu_qsbr_writer_perf,
161 (void *)1, enabled_core_ids[i]);
163 /* Wait for the writer thread */
164 rte_eal_wait_lcore(enabled_core_ids[i]);
167 /* Wait until all readers have exited */
168 rte_eal_mp_wait_lcore();
170 printf("Total RCU updates = %"PRIi64"\n", rte_atomic64_read(&updates));
171 printf("Cycles per %d updates: %"PRIi64"\n", RCU_SCALE_DOWN,
172 rte_atomic64_read(&update_cycles) /
173 (rte_atomic64_read(&updates) / RCU_SCALE_DOWN));
174 printf("Total RCU checks = %"PRIi64"\n", rte_atomic64_read(&checks));
175 printf("Cycles per %d checks: %"PRIi64"\n", RCU_SCALE_DOWN,
176 rte_atomic64_read(&check_cycles) /
177 (rte_atomic64_read(&checks) / RCU_SCALE_DOWN));
186 * Single writer, Multiple readers, Single QS variable
189 test_rcu_qsbr_rperf(void)
194 rte_atomic64_clear(&updates);
195 rte_atomic64_clear(&update_cycles);
197 __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
199 printf("\nPerf Test: %d Readers\n", num_cores);
201 if (all_registered == 1)
202 tmp_num_cores = num_cores;
204 tmp_num_cores = RTE_MAX_LCORE;
206 sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
207 t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
208 RTE_CACHE_LINE_SIZE);
209 /* QS variable is initialized */
210 rte_rcu_qsbr_init(t[0], tmp_num_cores);
212 /* Reader threads are launched */
213 for (i = 0; i < num_cores; i++)
214 rte_eal_remote_launch(test_rcu_qsbr_reader_perf, NULL,
215 enabled_core_ids[i]);
217 /* Wait until all readers have exited */
218 rte_eal_mp_wait_lcore();
220 printf("Total RCU updates = %"PRIi64"\n", rte_atomic64_read(&updates));
221 printf("Cycles per %d updates: %"PRIi64"\n", RCU_SCALE_DOWN,
222 rte_atomic64_read(&update_cycles) /
223 (rte_atomic64_read(&updates) / RCU_SCALE_DOWN));
232 * Multiple writer, Single QS variable, Non-blocking rcu_qsbr_check
235 test_rcu_qsbr_wperf(void)
239 rte_atomic64_clear(&checks);
240 rte_atomic64_clear(&check_cycles);
242 __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
244 printf("\nPerf test: %d Writers ('wait' in qsbr_check == false)\n",
247 /* Number of readers does not matter for QS variable in this test
248 * case as no reader will be registered.
250 sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
251 t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
252 RTE_CACHE_LINE_SIZE);
253 /* QS variable is initialized */
254 rte_rcu_qsbr_init(t[0], RTE_MAX_LCORE);
256 /* Writer threads are launched */
257 for (i = 0; i < num_cores; i++)
258 rte_eal_remote_launch(test_rcu_qsbr_writer_perf,
259 (void *)0, enabled_core_ids[i]);
261 /* Wait until all readers have exited */
262 rte_eal_mp_wait_lcore();
264 printf("Total RCU checks = %"PRIi64"\n", rte_atomic64_read(&checks));
265 printf("Cycles per %d checks: %"PRIi64"\n", RCU_SCALE_DOWN,
266 rte_atomic64_read(&check_cycles) /
267 (rte_atomic64_read(&checks) / RCU_SCALE_DOWN));
275 * RCU test cases using rte_hash data structure.
278 test_rcu_qsbr_hash_reader(void *arg)
280 struct rte_rcu_qsbr *temp;
281 struct rte_hash *hash = NULL;
283 uint64_t loop_cnt = 0;
284 uint64_t begin, cycles;
285 uint32_t thread_id = alloc_thread_id();
286 uint8_t read_type = (uint8_t)((uintptr_t)arg);
292 rte_rcu_qsbr_thread_register(temp, thread_id);
294 begin = rte_rdtsc_precise();
297 rte_rcu_qsbr_thread_online(temp, thread_id);
298 for (i = 0; i < TOTAL_ENTRY; i++) {
299 rte_rcu_qsbr_lock(temp, thread_id);
300 if (rte_hash_lookup_data(hash, keys + i,
301 (void **)&pdata) != -ENOENT) {
302 pdata[thread_id] = 0;
303 while (pdata[thread_id] < COUNTER_VALUE)
306 rte_rcu_qsbr_unlock(temp, thread_id);
308 /* Update quiescent state counter */
309 rte_rcu_qsbr_quiescent(temp, thread_id);
310 rte_rcu_qsbr_thread_offline(temp, thread_id);
312 } while (!writer_done);
314 cycles = rte_rdtsc_precise() - begin;
315 rte_atomic64_add(&update_cycles, cycles);
316 rte_atomic64_add(&updates, loop_cnt);
318 rte_rcu_qsbr_thread_unregister(temp, thread_id);
323 static struct rte_hash *init_hash(void)
326 struct rte_hash *hash = NULL;
328 snprintf(hash_name, 8, "hash");
329 struct rte_hash_parameters hash_params = {
330 .entries = TOTAL_ENTRY,
331 .key_len = sizeof(uint32_t),
332 .hash_func_init_val = 0,
333 .socket_id = rte_socket_id(),
334 .hash_func = rte_hash_crc,
336 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF,
340 hash = rte_hash_create(&hash_params);
342 printf("Hash create Failed\n");
346 for (i = 0; i < TOTAL_ENTRY; i++) {
347 hash_data[i] = rte_zmalloc(NULL,
348 sizeof(uint32_t) * RTE_MAX_LCORE, 0);
349 if (hash_data[i] == NULL) {
350 printf("No memory\n");
354 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
356 printf("No memory\n");
360 for (i = 0; i < TOTAL_ENTRY; i++)
363 for (i = 0; i < TOTAL_ENTRY; i++) {
364 if (rte_hash_add_key_data(hash, keys + i,
365 (void *)((uintptr_t)hash_data[i])) < 0) {
366 printf("Hash key add Failed #%d\n", i);
375 * Single writer, Single QS variable Single QSBR query, Blocking rcu_qsbr_check
378 test_rcu_qsbr_sw_sv_1qs(void)
380 uint64_t token, begin, cycles;
381 int i, j, tmp_num_cores, sz;
386 rte_atomic64_clear(&updates);
387 rte_atomic64_clear(&update_cycles);
388 rte_atomic64_clear(&checks);
389 rte_atomic64_clear(&check_cycles);
391 __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
393 printf("\nPerf test: 1 writer, %d readers, 1 QSBR variable, 1 QSBR Query, Blocking QSBR Check\n", num_cores);
395 if (all_registered == 1)
396 tmp_num_cores = num_cores;
398 tmp_num_cores = RTE_MAX_LCORE;
400 sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
401 t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
402 RTE_CACHE_LINE_SIZE);
403 /* QS variable is initialized */
404 rte_rcu_qsbr_init(t[0], tmp_num_cores);
406 /* Shared data structure created */
409 printf("Hash init failed\n");
413 /* Reader threads are launched */
414 for (i = 0; i < num_cores; i++)
415 rte_eal_remote_launch(test_rcu_qsbr_hash_reader, NULL,
416 enabled_core_ids[i]);
418 begin = rte_rdtsc_precise();
420 for (i = 0; i < TOTAL_ENTRY; i++) {
421 /* Delete elements from the shared data structure */
422 pos = rte_hash_del_key(h, keys + i);
424 printf("Delete key failed #%d\n", keys[i]);
427 /* Start the quiescent state query process */
428 token = rte_rcu_qsbr_start(t[0]);
430 /* Check the quiescent state status */
431 rte_rcu_qsbr_check(t[0], token, true);
432 for (j = 0; j < tmp_num_cores; j++) {
433 if (hash_data[i][j] != COUNTER_VALUE &&
434 hash_data[i][j] != 0) {
435 printf("Reader thread ID %u did not complete #%d = %d\n",
436 j, i, hash_data[i][j]);
441 if (rte_hash_free_key_with_position(h, pos) < 0) {
442 printf("Failed to free the key #%d\n", keys[i]);
445 rte_free(hash_data[i]);
449 cycles = rte_rdtsc_precise() - begin;
450 rte_atomic64_add(&check_cycles, cycles);
451 rte_atomic64_add(&checks, i);
455 /* Wait and check return value from reader threads */
456 for (i = 0; i < num_cores; i++)
457 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
462 printf("Following numbers include calls to rte_hash functions\n");
463 printf("Cycles per 1 update(online/update/offline): %"PRIi64"\n",
464 rte_atomic64_read(&update_cycles) /
465 rte_atomic64_read(&updates));
467 printf("Cycles per 1 check(start, check): %"PRIi64"\n\n",
468 rte_atomic64_read(&check_cycles) /
469 rte_atomic64_read(&checks));
477 /* Wait until all readers have exited */
478 rte_eal_mp_wait_lcore();
482 for (i = 0; i < TOTAL_ENTRY; i++)
483 rte_free(hash_data[i]);
492 * Single writer, Single QS variable, Single QSBR query,
493 * Non-blocking rcu_qsbr_check
496 test_rcu_qsbr_sw_sv_1qs_non_blocking(void)
498 uint64_t token, begin, cycles;
499 int i, j, ret, tmp_num_cores, sz;
504 printf("Perf test: 1 writer, %d readers, 1 QSBR variable, 1 QSBR Query, Non-Blocking QSBR check\n", num_cores);
506 __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
508 if (all_registered == 1)
509 tmp_num_cores = num_cores;
511 tmp_num_cores = RTE_MAX_LCORE;
513 sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
514 t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
515 RTE_CACHE_LINE_SIZE);
516 /* QS variable is initialized */
517 rte_rcu_qsbr_init(t[0], tmp_num_cores);
519 /* Shared data structure created */
522 printf("Hash init failed\n");
526 /* Reader threads are launched */
527 for (i = 0; i < num_cores; i++)
528 rte_eal_remote_launch(test_rcu_qsbr_hash_reader, NULL,
529 enabled_core_ids[i]);
531 begin = rte_rdtsc_precise();
533 for (i = 0; i < TOTAL_ENTRY; i++) {
534 /* Delete elements from the shared data structure */
535 pos = rte_hash_del_key(h, keys + i);
537 printf("Delete key failed #%d\n", keys[i]);
540 /* Start the quiescent state query process */
541 token = rte_rcu_qsbr_start(t[0]);
543 /* Check the quiescent state status */
545 ret = rte_rcu_qsbr_check(t[0], token, false);
547 for (j = 0; j < tmp_num_cores; j++) {
548 if (hash_data[i][j] != COUNTER_VALUE &&
549 hash_data[i][j] != 0) {
550 printf("Reader thread ID %u did not complete #%d = %d\n",
551 j, i, hash_data[i][j]);
556 if (rte_hash_free_key_with_position(h, pos) < 0) {
557 printf("Failed to free the key #%d\n", keys[i]);
560 rte_free(hash_data[i]);
564 cycles = rte_rdtsc_precise() - begin;
565 rte_atomic64_add(&check_cycles, cycles);
566 rte_atomic64_add(&checks, i);
569 /* Wait and check return value from reader threads */
570 for (i = 0; i < num_cores; i++)
571 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
576 printf("Following numbers include calls to rte_hash functions\n");
577 printf("Cycles per 1 update(online/update/offline): %"PRIi64"\n",
578 rte_atomic64_read(&update_cycles) /
579 rte_atomic64_read(&updates));
581 printf("Cycles per 1 check(start, check): %"PRIi64"\n\n",
582 rte_atomic64_read(&check_cycles) /
583 rte_atomic64_read(&checks));
591 /* Wait until all readers have exited */
592 rte_eal_mp_wait_lcore();
596 for (i = 0; i < TOTAL_ENTRY; i++)
597 rte_free(hash_data[i]);
605 test_rcu_qsbr_main(void)
609 rte_atomic64_init(&updates);
610 rte_atomic64_init(&update_cycles);
611 rte_atomic64_init(&checks);
612 rte_atomic64_init(&check_cycles);
615 RTE_LCORE_FOREACH_SLAVE(core_id) {
616 enabled_core_ids[num_cores] = core_id;
620 printf("Number of cores provided = %d\n", num_cores);
622 printf("Test failed! Need 2 or more cores\n");
626 printf("Perf test with all reader threads registered\n");
627 printf("--------------------------------------------\n");
630 if (test_rcu_qsbr_perf() < 0)
633 if (test_rcu_qsbr_rperf() < 0)
636 if (test_rcu_qsbr_wperf() < 0)
639 if (test_rcu_qsbr_sw_sv_1qs() < 0)
642 if (test_rcu_qsbr_sw_sv_1qs_non_blocking() < 0)
645 /* Make sure the actual number of cores provided is less than
646 * RTE_MAX_LCORE. This will allow for some threads not
647 * to be registered on the QS variable.
649 if (num_cores >= RTE_MAX_LCORE) {
650 printf("Test failed! number of cores provided should be less than %d\n",
655 printf("Perf test with some of reader threads registered\n");
656 printf("------------------------------------------------\n");
659 if (test_rcu_qsbr_perf() < 0)
662 if (test_rcu_qsbr_rperf() < 0)
665 if (test_rcu_qsbr_wperf() < 0)
668 if (test_rcu_qsbr_sw_sv_1qs() < 0)
671 if (test_rcu_qsbr_sw_sv_1qs_non_blocking() < 0)
682 REGISTER_TEST_COMMAND(rcu_qsbr_perf_autotest, test_rcu_qsbr_main);