1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
12 #include <rte_common.h>
13 #include <rte_memory.h>
14 #include <rte_per_lcore.h>
15 #include <rte_launch.h>
16 #include <rte_rwlock.h>
18 #include <rte_lcore.h>
19 #include <rte_cycles.h>
26 * Provides UT for rte_rwlock API.
27 * Main concern is on functional testing, but also provides some
28 * performance measurements.
29 * Obviously for proper testing need to be executed with more than one lcore.
36 static rte_rwlock_t sl;
37 static rte_rwlock_t sl_tab[RTE_MAX_LCORE];
38 static uint32_t synchro;
50 uint8_t u8[RTE_CACHE_LINE_SIZE];
51 uint64_t u64[RTE_CACHE_LINE_SIZE / sizeof(uint64_t)];
53 } __rte_cache_aligned try_rwlock_data;
55 struct try_rwlock_lcore {
63 } __rte_cache_aligned;
65 static struct try_rwlock_lcore try_lcore_data[RTE_MAX_LCORE];
68 test_rwlock_per_core(__rte_unused void *arg)
70 rte_rwlock_write_lock(&sl);
71 printf("Global write lock taken on core %u\n", rte_lcore_id());
72 rte_rwlock_write_unlock(&sl);
74 rte_rwlock_write_lock(&sl_tab[rte_lcore_id()]);
75 printf("Hello from core %u !\n", rte_lcore_id());
76 rte_rwlock_write_unlock(&sl_tab[rte_lcore_id()]);
78 rte_rwlock_read_lock(&sl);
79 printf("Global read lock taken on core %u\n", rte_lcore_id());
81 printf("Release global read lock on core %u\n", rte_lcore_id());
82 rte_rwlock_read_unlock(&sl);
87 static rte_rwlock_t lk = RTE_RWLOCK_INITIALIZER;
88 static volatile uint64_t rwlock_data;
89 static uint64_t time_count[RTE_MAX_LCORE] = {0};
91 #define MAX_LOOP 10000
92 #define TEST_RWLOCK_DEBUG 0
95 load_loop_fn(__rte_unused void *arg)
97 uint64_t time_diff = 0, begin;
98 uint64_t hz = rte_get_timer_hz();
100 const unsigned int lcore = rte_lcore_id();
102 /* wait synchro for workers */
103 if (lcore != rte_get_main_lcore())
104 rte_wait_until_equal_32(&synchro, 1, __ATOMIC_RELAXED);
106 begin = rte_rdtsc_precise();
107 while (lcount < MAX_LOOP) {
108 rte_rwlock_write_lock(&lk);
110 rte_rwlock_write_unlock(&lk);
112 rte_rwlock_read_lock(&lk);
113 if (TEST_RWLOCK_DEBUG && !(lcount % 100))
114 printf("Core [%u] rwlock_data = %"PRIu64"\n",
116 rte_rwlock_read_unlock(&lk);
119 /* delay to make lock duty cycle slightly realistic */
123 time_diff = rte_rdtsc_precise() - begin;
124 time_count[lcore] = time_diff * 1000000 / hz;
129 test_rwlock_perf(void)
134 printf("\nRwlock Perf Test on %u cores...\n", rte_lcore_count());
136 /* clear synchro and start workers */
137 __atomic_store_n(&synchro, 0, __ATOMIC_RELAXED);
138 if (rte_eal_mp_remote_launch(load_loop_fn, NULL, SKIP_MAIN) < 0)
141 /* start synchro and launch test on main */
142 __atomic_store_n(&synchro, 1, __ATOMIC_RELAXED);
145 rte_eal_mp_wait_lcore();
147 RTE_LCORE_FOREACH(i) {
148 printf("Core [%u] cost time = %"PRIu64" us\n",
150 total += time_count[i];
153 printf("Total cost time = %"PRIu64" us\n", total);
154 memset(time_count, 0, sizeof(time_count));
160 * - There is a global rwlock and a table of rwlocks (one per lcore).
162 * - The test function takes all of these locks and launches the
163 * ``test_rwlock_per_core()`` function on each core (except the main).
165 * - The function takes the global write lock, display something,
166 * then releases the global lock.
167 * - Then, it takes the per-lcore write lock, display something, and
168 * releases the per-core lock.
169 * - Finally, a read lock is taken during 100 ms, then released.
171 * - The main function unlocks the per-lcore locks sequentially and
172 * waits between each lock. This triggers the display of a message
173 * for each core, in the correct order.
175 * Then, it tries to take the global write lock and display the last
176 * message. The autotest script checks that the message order is correct.
183 rte_rwlock_init(&sl);
184 for (i = 0; i < RTE_MAX_LCORE; i++)
185 rte_rwlock_init(&sl_tab[i]);
187 rte_rwlock_write_lock(&sl);
189 RTE_LCORE_FOREACH_WORKER(i) {
190 rte_rwlock_write_lock(&sl_tab[i]);
191 rte_eal_remote_launch(test_rwlock_per_core, NULL, i);
194 rte_rwlock_write_unlock(&sl);
196 RTE_LCORE_FOREACH_WORKER(i) {
197 rte_rwlock_write_unlock(&sl_tab[i]);
201 rte_rwlock_write_lock(&sl);
202 /* this message should be the last message of test */
203 printf("Global write lock taken on main core %u\n", rte_lcore_id());
204 rte_rwlock_write_unlock(&sl);
206 rte_eal_mp_wait_lcore();
208 if (test_rwlock_perf() < 0)
215 try_read(uint32_t lc)
220 rc = rte_rwlock_read_trylock(&try_rwlock_data.lock);
224 for (i = 0; i != RTE_DIM(try_rwlock_data.data.u64); i++) {
226 /* race condition occurred, lock doesn't work properly */
227 if (try_rwlock_data.data.u64[i] != 0) {
228 printf("%s(%u) error: unexpected data pattern\n",
230 rte_memdump(stdout, NULL,
231 (void *)(uintptr_t)&try_rwlock_data.data,
232 sizeof(try_rwlock_data.data));
238 rte_rwlock_read_unlock(&try_rwlock_data.lock);
243 try_write(uint32_t lc)
248 v = RTE_MAX(lc % UINT8_MAX, 1U);
250 rc = rte_rwlock_write_trylock(&try_rwlock_data.lock);
254 /* update by bytes in reverse order */
255 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) {
257 /* race condition occurred, lock doesn't work properly */
258 if (try_rwlock_data.data.u8[i] != 0) {
259 printf("%s:%d(%u) error: unexpected data pattern\n",
260 __func__, __LINE__, lc);
261 rte_memdump(stdout, NULL,
262 (void *)(uintptr_t)&try_rwlock_data.data,
263 sizeof(try_rwlock_data.data));
268 try_rwlock_data.data.u8[i] = v;
271 /* restore by bytes in reverse order */
272 for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) {
274 /* race condition occurred, lock doesn't work properly */
275 if (try_rwlock_data.data.u8[i] != v) {
276 printf("%s:%d(%u) error: unexpected data pattern\n",
277 __func__, __LINE__, lc);
278 rte_memdump(stdout, NULL,
279 (void *)(uintptr_t)&try_rwlock_data.data,
280 sizeof(try_rwlock_data.data));
285 try_rwlock_data.data.u8[i] = 0;
288 rte_rwlock_write_unlock(&try_rwlock_data.lock);
293 try_read_lcore(__rte_unused void *data)
297 uint64_t ftm, stm, tm;
298 struct try_rwlock_lcore *lcd;
301 lcd = try_lcore_data + lc;
302 lcd->type = LC_TYPE_RDLOCK;
304 ftm = try_rwlock_data.tick;
305 stm = rte_get_timer_cycles();
308 for (i = 0; i != ITER_NUM; i++) {
312 else if (rc == -EBUSY)
318 tm = rte_get_timer_cycles() - stm;
319 } while (tm < ftm && rc == 0);
327 try_write_lcore(__rte_unused void *data)
331 uint64_t ftm, stm, tm;
332 struct try_rwlock_lcore *lcd;
335 lcd = try_lcore_data + lc;
336 lcd->type = LC_TYPE_WRLOCK;
338 ftm = try_rwlock_data.tick;
339 stm = rte_get_timer_cycles();
342 for (i = 0; i != ITER_NUM; i++) {
346 else if (rc == -EBUSY)
352 tm = rte_get_timer_cycles() - stm;
353 } while (tm < ftm && rc == 0);
361 print_try_lcore_stats(const struct try_rwlock_lcore *tlc, uint32_t lc)
365 f = RTE_MAX(tlc->stat.fail, 1ULL);
366 s = RTE_MAX(tlc->stat.success, 1ULL);
368 printf("try_lcore_data[%u]={\n"
371 "\tfail=%" PRIu64 ",\n"
372 "\tsuccess=%" PRIu64 ",\n"
373 "\tcycles=%" PRIu64 ",\n"
374 "\tcycles/op=%#Lf,\n"
375 "\tcycles/success=%#Lf,\n"
376 "\tsuccess/fail=%#Lf,\n"
380 tlc->type == LC_TYPE_RDLOCK ? "RDLOCK" : "WRLOCK",
384 (long double)tlc->stat.tick /
385 (tlc->stat.fail + tlc->stat.success),
386 (long double)tlc->stat.tick / s,
387 (long double)tlc->stat.success / f);
391 collect_try_lcore_stats(struct try_rwlock_lcore *tlc,
392 const struct try_rwlock_lcore *lc)
394 tlc->stat.tick += lc->stat.tick;
395 tlc->stat.fail += lc->stat.fail;
396 tlc->stat.success += lc->stat.success;
400 * Process collected results:
402 * - collect and print statistics
405 process_try_lcore_stats(void)
409 struct try_rwlock_lcore rlc, wlc;
411 memset(&rlc, 0, sizeof(rlc));
412 memset(&wlc, 0, sizeof(wlc));
414 rlc.type = LC_TYPE_RDLOCK;
415 wlc.type = LC_TYPE_WRLOCK;
420 RTE_LCORE_FOREACH(lc) {
421 rc |= try_lcore_data[lc].rc;
422 if (try_lcore_data[lc].type == LC_TYPE_RDLOCK) {
423 collect_try_lcore_stats(&rlc, try_lcore_data + lc);
426 collect_try_lcore_stats(&wlc, try_lcore_data + lc);
432 RTE_LCORE_FOREACH(lc)
433 print_try_lcore_stats(try_lcore_data + lc, lc);
436 printf("aggregated stats for %u RDLOCK cores:\n", rd);
437 print_try_lcore_stats(&rlc, rd);
441 printf("aggregated stats for %u WRLOCK cores:\n", wr);
442 print_try_lcore_stats(&wlc, wr);
452 memset(&try_lcore_data, 0, sizeof(try_lcore_data));
453 memset(&try_rwlock_data, 0, sizeof(try_rwlock_data));
454 try_rwlock_data.tick = TEST_SEC * rte_get_tsc_hz();
457 /* all lcores grab RDLOCK */
459 try_rwlock_test_rda(void)
463 /* start read test on all available lcores */
464 rte_eal_mp_remote_launch(try_read_lcore, NULL, CALL_MAIN);
465 rte_eal_mp_wait_lcore();
467 return process_try_lcore_stats();
470 /* all worker lcores grab RDLOCK, main one grabs WRLOCK */
472 try_rwlock_test_rds_wrm(void)
476 rte_eal_mp_remote_launch(try_read_lcore, NULL, SKIP_MAIN);
477 try_write_lcore(NULL);
478 rte_eal_mp_wait_lcore();
480 return process_try_lcore_stats();
483 /* main and even worker lcores grab RDLOCK, odd lcores grab WRLOCK */
485 try_rwlock_test_rde_wro(void)
491 mlc = rte_get_main_lcore();
493 RTE_LCORE_FOREACH(lc) {
496 rte_eal_remote_launch(try_read_lcore,
499 rte_eal_remote_launch(try_write_lcore,
503 try_read_lcore(NULL);
504 rte_eal_mp_wait_lcore();
506 return process_try_lcore_stats();
509 REGISTER_TEST_COMMAND(rwlock_test1_autotest, rwlock_test1);
510 REGISTER_TEST_COMMAND(rwlock_rda_autotest, try_rwlock_test_rda);
511 REGISTER_TEST_COMMAND(rwlock_rds_wrm_autotest, try_rwlock_test_rds_wrm);
512 REGISTER_TEST_COMMAND(rwlock_rde_wro_autotest, try_rwlock_test_rde_wro);