1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
12 #include <rte_common.h>
13 #include <rte_memory.h>
14 #include <rte_per_lcore.h>
15 #include <rte_launch.h>
16 #include <rte_atomic.h>
17 #include <rte_rwlock.h>
19 #include <rte_lcore.h>
20 #include <rte_cycles.h>
27 * Provides UT for rte_rwlock API.
28 * Main concern is on functional testing, but also provides some
29 * performance measurements.
30 * Obviously for proper testing need to be executed with more than one lcore.
37 static rte_rwlock_t sl;
38 static rte_rwlock_t sl_tab[RTE_MAX_LCORE];
39 static rte_atomic32_t synchro;
51 uint8_t u8[RTE_CACHE_LINE_SIZE];
52 uint64_t u64[RTE_CACHE_LINE_SIZE / sizeof(uint64_t)];
54 } __rte_cache_aligned try_rwlock_data;
56 struct try_rwlock_lcore {
64 } __rte_cache_aligned;
66 static struct try_rwlock_lcore try_lcore_data[RTE_MAX_LCORE];
69 test_rwlock_per_core(__rte_unused void *arg)
71 rte_rwlock_write_lock(&sl);
72 printf("Global write lock taken on core %u\n", rte_lcore_id());
73 rte_rwlock_write_unlock(&sl);
75 rte_rwlock_write_lock(&sl_tab[rte_lcore_id()]);
76 printf("Hello from core %u !\n", rte_lcore_id());
77 rte_rwlock_write_unlock(&sl_tab[rte_lcore_id()]);
79 rte_rwlock_read_lock(&sl);
80 printf("Global read lock taken on core %u\n", rte_lcore_id());
82 printf("Release global read lock on core %u\n", rte_lcore_id());
83 rte_rwlock_read_unlock(&sl);
88 static rte_rwlock_t lk = RTE_RWLOCK_INITIALIZER;
89 static volatile uint64_t rwlock_data;
90 static uint64_t time_count[RTE_MAX_LCORE] = {0};
92 #define MAX_LOOP 10000
93 #define TEST_RWLOCK_DEBUG 0
96 load_loop_fn(__rte_unused void *arg)
98 uint64_t time_diff = 0, begin;
99 uint64_t hz = rte_get_timer_hz();
101 const unsigned int lcore = rte_lcore_id();
103 /* wait synchro for workers */
104 if (lcore != rte_get_main_lcore())
105 while (rte_atomic32_read(&synchro) == 0)
108 begin = rte_rdtsc_precise();
109 while (lcount < MAX_LOOP) {
110 rte_rwlock_write_lock(&lk);
112 rte_rwlock_write_unlock(&lk);
114 rte_rwlock_read_lock(&lk);
115 if (TEST_RWLOCK_DEBUG && !(lcount % 100))
116 printf("Core [%u] rwlock_data = %"PRIu64"\n",
118 rte_rwlock_read_unlock(&lk);
121 /* delay to make lock duty cycle slightly realistic */
125 time_diff = rte_rdtsc_precise() - begin;
126 time_count[lcore] = time_diff * 1000000 / hz;
131 test_rwlock_perf(void)
136 printf("\nRwlock Perf Test on %u cores...\n", rte_lcore_count());
138 /* clear synchro and start workers */
139 rte_atomic32_set(&synchro, 0);
140 if (rte_eal_mp_remote_launch(load_loop_fn, NULL, SKIP_MAIN) < 0)
143 /* start synchro and launch test on main */
144 rte_atomic32_set(&synchro, 1);
147 rte_eal_mp_wait_lcore();
149 RTE_LCORE_FOREACH(i) {
150 printf("Core [%u] cost time = %"PRIu64" us\n",
152 total += time_count[i];
155 printf("Total cost time = %"PRIu64" us\n", total);
156 memset(time_count, 0, sizeof(time_count));
162 * - There is a global rwlock and a table of rwlocks (one per lcore).
164 * - The test function takes all of these locks and launches the
165 * ``test_rwlock_per_core()`` function on each core (except the main).
167 * - The function takes the global write lock, display something,
168 * then releases the global lock.
169 * - Then, it takes the per-lcore write lock, display something, and
170 * releases the per-core lock.
171 * - Finally, a read lock is taken during 100 ms, then released.
173 * - The main function unlocks the per-lcore locks sequentially and
174 * waits between each lock. This triggers the display of a message
175 * for each core, in the correct order.
177 * Then, it tries to take the global write lock and display the last
178 * message. The autotest script checks that the message order is correct.
185 rte_rwlock_init(&sl);
186 for (i = 0; i < RTE_MAX_LCORE; i++)
187 rte_rwlock_init(&sl_tab[i]);
189 rte_rwlock_write_lock(&sl);
191 RTE_LCORE_FOREACH_WORKER(i) {
192 rte_rwlock_write_lock(&sl_tab[i]);
193 rte_eal_remote_launch(test_rwlock_per_core, NULL, i);
196 rte_rwlock_write_unlock(&sl);
198 RTE_LCORE_FOREACH_WORKER(i) {
199 rte_rwlock_write_unlock(&sl_tab[i]);
203 rte_rwlock_write_lock(&sl);
204 /* this message should be the last message of test */
205 printf("Global write lock taken on main core %u\n", rte_lcore_id());
206 rte_rwlock_write_unlock(&sl);
208 rte_eal_mp_wait_lcore();
210 if (test_rwlock_perf() < 0)
217 try_read(uint32_t lc)
222 rc = rte_rwlock_read_trylock(&try_rwlock_data.lock);
226 for (i = 0; i != RTE_DIM(try_rwlock_data.data.u64); i++) {
228 /* race condition occurred, lock doesn't work properly */
229 if (try_rwlock_data.data.u64[i] != 0) {
230 printf("%s(%u) error: unexpected data pattern\n",
232 rte_memdump(stdout, NULL,
233 (void *)(uintptr_t)&try_rwlock_data.data,
234 sizeof(try_rwlock_data.data));
240 rte_rwlock_read_unlock(&try_rwlock_data.lock);
245 try_write(uint32_t lc)
250 v = RTE_MAX(lc % UINT8_MAX, 1U);
252 rc = rte_rwlock_write_trylock(&try_rwlock_data.lock);
256 /* update by bytes in reverse order */
257 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) {
259 /* race condition occurred, lock doesn't work properly */
260 if (try_rwlock_data.data.u8[i] != 0) {
261 printf("%s:%d(%u) error: unexpected data pattern\n",
262 __func__, __LINE__, lc);
263 rte_memdump(stdout, NULL,
264 (void *)(uintptr_t)&try_rwlock_data.data,
265 sizeof(try_rwlock_data.data));
270 try_rwlock_data.data.u8[i] = v;
273 /* restore by bytes in reverse order */
274 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) {
276 /* race condition occurred, lock doesn't work properly */
277 if (try_rwlock_data.data.u8[i] != v) {
278 printf("%s:%d(%u) error: unexpected data pattern\n",
279 __func__, __LINE__, lc);
280 rte_memdump(stdout, NULL,
281 (void *)(uintptr_t)&try_rwlock_data.data,
282 sizeof(try_rwlock_data.data));
287 try_rwlock_data.data.u8[i] = 0;
290 rte_rwlock_write_unlock(&try_rwlock_data.lock);
295 try_read_lcore(__rte_unused void *data)
299 uint64_t ftm, stm, tm;
300 struct try_rwlock_lcore *lcd;
303 lcd = try_lcore_data + lc;
304 lcd->type = LC_TYPE_RDLOCK;
306 ftm = try_rwlock_data.tick;
307 stm = rte_get_timer_cycles();
310 for (i = 0; i != ITER_NUM; i++) {
314 else if (rc == -EBUSY)
320 tm = rte_get_timer_cycles() - stm;
321 } while (tm < ftm && rc == 0);
329 try_write_lcore(__rte_unused void *data)
333 uint64_t ftm, stm, tm;
334 struct try_rwlock_lcore *lcd;
337 lcd = try_lcore_data + lc;
338 lcd->type = LC_TYPE_WRLOCK;
340 ftm = try_rwlock_data.tick;
341 stm = rte_get_timer_cycles();
344 for (i = 0; i != ITER_NUM; i++) {
348 else if (rc == -EBUSY)
354 tm = rte_get_timer_cycles() - stm;
355 } while (tm < ftm && rc == 0);
363 print_try_lcore_stats(const struct try_rwlock_lcore *tlc, uint32_t lc)
367 f = RTE_MAX(tlc->stat.fail, 1ULL);
368 s = RTE_MAX(tlc->stat.success, 1ULL);
370 printf("try_lcore_data[%u]={\n"
373 "\tfail=%" PRIu64 ",\n"
374 "\tsuccess=%" PRIu64 ",\n"
375 "\tcycles=%" PRIu64 ",\n"
376 "\tcycles/op=%#Lf,\n"
377 "\tcycles/success=%#Lf,\n"
378 "\tsuccess/fail=%#Lf,\n"
382 tlc->type == LC_TYPE_RDLOCK ? "RDLOCK" : "WRLOCK",
386 (long double)tlc->stat.tick /
387 (tlc->stat.fail + tlc->stat.success),
388 (long double)tlc->stat.tick / s,
389 (long double)tlc->stat.success / f);
393 collect_try_lcore_stats(struct try_rwlock_lcore *tlc,
394 const struct try_rwlock_lcore *lc)
396 tlc->stat.tick += lc->stat.tick;
397 tlc->stat.fail += lc->stat.fail;
398 tlc->stat.success += lc->stat.success;
402 * Process collected results:
404 * - collect and print statistics
407 process_try_lcore_stats(void)
411 struct try_rwlock_lcore rlc, wlc;
413 memset(&rlc, 0, sizeof(rlc));
414 memset(&wlc, 0, sizeof(wlc));
416 rlc.type = LC_TYPE_RDLOCK;
417 wlc.type = LC_TYPE_WRLOCK;
422 RTE_LCORE_FOREACH(lc) {
423 rc |= try_lcore_data[lc].rc;
424 if (try_lcore_data[lc].type == LC_TYPE_RDLOCK) {
425 collect_try_lcore_stats(&rlc, try_lcore_data + lc);
428 collect_try_lcore_stats(&wlc, try_lcore_data + lc);
434 RTE_LCORE_FOREACH(lc)
435 print_try_lcore_stats(try_lcore_data + lc, lc);
438 printf("aggregated stats for %u RDLOCK cores:\n", rd);
439 print_try_lcore_stats(&rlc, rd);
443 printf("aggregated stats for %u WRLOCK cores:\n", wr);
444 print_try_lcore_stats(&wlc, wr);
454 memset(&try_lcore_data, 0, sizeof(try_lcore_data));
455 memset(&try_rwlock_data, 0, sizeof(try_rwlock_data));
456 try_rwlock_data.tick = TEST_SEC * rte_get_tsc_hz();
459 /* all lcores grab RDLOCK */
461 try_rwlock_test_rda(void)
465 /* start read test on all available lcores */
466 rte_eal_mp_remote_launch(try_read_lcore, NULL, CALL_MAIN);
467 rte_eal_mp_wait_lcore();
469 return process_try_lcore_stats();
472 /* all worker lcores grab RDLOCK, main one grabs WRLOCK */
474 try_rwlock_test_rds_wrm(void)
478 rte_eal_mp_remote_launch(try_read_lcore, NULL, SKIP_MAIN);
479 try_write_lcore(NULL);
480 rte_eal_mp_wait_lcore();
482 return process_try_lcore_stats();
485 /* main and even worker lcores grab RDLOCK, odd lcores grab WRLOCK */
487 try_rwlock_test_rde_wro(void)
493 mlc = rte_get_main_lcore();
495 RTE_LCORE_FOREACH(lc) {
498 rte_eal_remote_launch(try_read_lcore,
501 rte_eal_remote_launch(try_write_lcore,
505 try_read_lcore(NULL);
506 rte_eal_mp_wait_lcore();
508 return process_try_lcore_stats();
517 static const struct {
522 .name = "rwlock_test1",
523 .ftst = rwlock_test1,
526 .name = "try_rwlock_test_rda",
527 .ftst = try_rwlock_test_rda,
530 .name = "try_rwlock_test_rds_wrm",
531 .ftst = try_rwlock_test_rds_wrm,
534 .name = "try_rwlock_test_rde_wro",
535 .ftst = try_rwlock_test_rde_wro,
540 for (i = 0; i != RTE_DIM(test); i++) {
541 printf("starting test %s;\n", test[i].name);
543 printf("test %s completed with status %d\n", test[i].name, rc);
550 REGISTER_TEST_COMMAND(rwlock_autotest, test_rwlock);
552 /* subtests used in meson for CI */
553 REGISTER_TEST_COMMAND(rwlock_test1_autotest, rwlock_test1);
554 REGISTER_TEST_COMMAND(rwlock_rda_autotest, try_rwlock_test_rda);
555 REGISTER_TEST_COMMAND(rwlock_rds_wrm_autotest, try_rwlock_test_rds_wrm);
556 REGISTER_TEST_COMMAND(rwlock_rde_wro_autotest, try_rwlock_test_rde_wro);