1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 * Copyright(c) 2019 Arm Limited
11 #include <rte_memory.h>
12 #include <rte_per_lcore.h>
13 #include <rte_launch.h>
14 #include <rte_atomic.h>
16 #include <rte_lcore.h>
24 * - The main test function performs four subtests. The first test
25 * checks that the usual inc/dec/add/sub functions are working
28 * - Initialize 16-bit, 32-bit and 64-bit atomic variables to specific
31 * - These variables are incremented and decremented on each core at
32 * the same time in ``test_atomic_usual()``.
34 * - The function checks that once all lcores finish their function,
35 * the value of the atomic variables are still the same.
37 * - The second test verifies the behavior of "test and set" functions.
39 * - Initialize 16-bit, 32-bit and 64-bit atomic variables to zero.
41 * - Invoke ``test_atomic_tas()`` on each lcore: before doing anything
42 * else. The cores are waiting a synchro using ``while
43 * (rte_atomic32_read(&val) == 0)`` which is triggered by the main test
44 * function. Then all cores do a
45 * ``rte_atomicXX_test_and_set()`` at the same time. If it is successful,
46 * it increments another atomic counter.
48 * - The main function checks that the atomic counter was incremented
49 * twice only (one for 16-bit, one for 32-bit and one for 64-bit values).
51 * - Test "add/sub and return"
53 * - Initialize 16-bit, 32-bit and 64-bit atomic variables to zero.
55 * - Invoke ``test_atomic_addsub_return()`` on each lcore. Before doing
56 * anything else, the cores are waiting a synchro. Each lcore does
57 * this operation several times::
59 * tmp = rte_atomicXX_add_return(&a, 1);
60 * atomic_add(&count, tmp);
61 * tmp = rte_atomicXX_sub_return(&a, 1);
62 * atomic_sub(&count, tmp+1);
64 * - At the end of the test, the *count* value must be 0.
66 * - Test "128-bit compare and swap" (aarch64 and x86_64 only)
68 * - Initialize 128-bit atomic variables to zero.
70 * - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing
71 * anything else, the cores are waiting a synchro. Each lcore does
72 * these compare and swap (CAS) operations several times::
74 * Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
75 * Released CAS update counter.val[0] + 2; counter.val[1] + 1;
76 * Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
77 * Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
79 * - At the end of the test, the *count128* first 64-bit value and
80 * second 64-bit value differ by the total iterations.
83 #define NUM_ATOMIC_TYPES 3
87 static rte_atomic16_t a16;
88 static rte_atomic32_t a32;
89 static rte_atomic64_t a64;
90 static rte_atomic64_t count;
91 static rte_atomic32_t synchro;
94 test_atomic_usual(__attribute__((unused)) void *arg)
98 while (rte_atomic32_read(&synchro) == 0)
101 for (i = 0; i < N; i++)
102 rte_atomic16_inc(&a16);
103 for (i = 0; i < N; i++)
104 rte_atomic16_dec(&a16);
105 for (i = 0; i < (N / 5); i++)
106 rte_atomic16_add(&a16, 5);
107 for (i = 0; i < (N / 5); i++)
108 rte_atomic16_sub(&a16, 5);
110 for (i = 0; i < N; i++)
111 rte_atomic32_inc(&a32);
112 for (i = 0; i < N; i++)
113 rte_atomic32_dec(&a32);
114 for (i = 0; i < (N / 5); i++)
115 rte_atomic32_add(&a32, 5);
116 for (i = 0; i < (N / 5); i++)
117 rte_atomic32_sub(&a32, 5);
119 for (i = 0; i < N; i++)
120 rte_atomic64_inc(&a64);
121 for (i = 0; i < N; i++)
122 rte_atomic64_dec(&a64);
123 for (i = 0; i < (N / 5); i++)
124 rte_atomic64_add(&a64, 5);
125 for (i = 0; i < (N / 5); i++)
126 rte_atomic64_sub(&a64, 5);
132 test_atomic_tas(__attribute__((unused)) void *arg)
134 while (rte_atomic32_read(&synchro) == 0)
137 if (rte_atomic16_test_and_set(&a16))
138 rte_atomic64_inc(&count);
139 if (rte_atomic32_test_and_set(&a32))
140 rte_atomic64_inc(&count);
141 if (rte_atomic64_test_and_set(&a64))
142 rte_atomic64_inc(&count);
148 test_atomic_addsub_and_return(__attribute__((unused)) void *arg)
155 while (rte_atomic32_read(&synchro) == 0)
158 for (i = 0; i < N; i++) {
159 tmp16 = rte_atomic16_add_return(&a16, 1);
160 rte_atomic64_add(&count, tmp16);
162 tmp16 = rte_atomic16_sub_return(&a16, 1);
163 rte_atomic64_sub(&count, tmp16+1);
165 tmp32 = rte_atomic32_add_return(&a32, 1);
166 rte_atomic64_add(&count, tmp32);
168 tmp32 = rte_atomic32_sub_return(&a32, 1);
169 rte_atomic64_sub(&count, tmp32+1);
171 tmp64 = rte_atomic64_add_return(&a64, 1);
172 rte_atomic64_add(&count, tmp64);
174 tmp64 = rte_atomic64_sub_return(&a64, 1);
175 rte_atomic64_sub(&count, tmp64+1);
182 * rte_atomic32_inc_and_test() would increase a 32 bits counter by one and then
183 * test if that counter is equal to 0. It would return true if the counter is 0
184 * and false if the counter is not 0. rte_atomic64_inc_and_test() could do the
185 * same thing but for a 64 bits counter.
186 * Here checks that if the 32/64 bits counter is equal to 0 after being atomically
187 * increased by one. If it is, increase the variable of "count" by one which would
188 * be checked as the result later.
192 test_atomic_inc_and_test(__attribute__((unused)) void *arg)
194 while (rte_atomic32_read(&synchro) == 0)
197 if (rte_atomic16_inc_and_test(&a16)) {
198 rte_atomic64_inc(&count);
200 if (rte_atomic32_inc_and_test(&a32)) {
201 rte_atomic64_inc(&count);
203 if (rte_atomic64_inc_and_test(&a64)) {
204 rte_atomic64_inc(&count);
211 * rte_atomicXX_dec_and_test() should decrease a 32 bits counter by one and then
212 * test if that counter is equal to 0. It should return true if the counter is 0
213 * and false if the counter is not 0.
214 * This test checks if the counter is equal to 0 after being atomically
215 * decreased by one. If it is, increase the value of "count" by one which is to
216 * be checked as the result later.
219 test_atomic_dec_and_test(__attribute__((unused)) void *arg)
221 while (rte_atomic32_read(&synchro) == 0)
224 if (rte_atomic16_dec_and_test(&a16))
225 rte_atomic64_inc(&count);
227 if (rte_atomic32_dec_and_test(&a32))
228 rte_atomic64_inc(&count);
230 if (rte_atomic64_dec_and_test(&a64))
231 rte_atomic64_inc(&count);
236 #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
237 static rte_int128_t count128;
240 * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
241 * bits by 2 and the second 64 bits by 1 in this test. It should return true
242 * if the compare exchange operation is successful.
243 * This test repeats 128 bits compare and swap operations N rounds. In each
244 * iteration it runs compare and swap operation with different memory models.
247 test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
249 rte_int128_t expected;
253 while (rte_atomic32_read(&synchro) == 0)
258 for (i = 0; i < N; i++) {
260 rte_int128_t desired;
262 desired.val[0] = expected.val[0] + 2;
263 desired.val[1] = expected.val[1] + 1;
265 success = rte_atomic128_cmp_exchange(&count128,
266 &expected, &desired, 1,
267 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
268 } while (success == 0);
271 rte_int128_t desired;
273 desired.val[0] = expected.val[0] + 2;
274 desired.val[1] = expected.val[1] + 1;
276 success = rte_atomic128_cmp_exchange(&count128,
277 &expected, &desired, 1,
278 __ATOMIC_RELEASE, __ATOMIC_RELAXED);
279 } while (success == 0);
282 rte_int128_t desired;
284 desired.val[0] = expected.val[0] + 2;
285 desired.val[1] = expected.val[1] + 1;
287 success = rte_atomic128_cmp_exchange(&count128,
288 &expected, &desired, 1,
289 __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
290 } while (success == 0);
293 rte_int128_t desired;
295 desired.val[0] = expected.val[0] + 2;
296 desired.val[1] = expected.val[1] + 1;
298 success = rte_atomic128_cmp_exchange(&count128,
299 &expected, &desired, 1,
300 __ATOMIC_RELAXED, __ATOMIC_RELAXED);
301 } while (success == 0);
311 rte_atomic16_init(&a16);
312 rte_atomic32_init(&a32);
313 rte_atomic64_init(&a64);
314 rte_atomic64_init(&count);
315 rte_atomic32_init(&synchro);
317 rte_atomic16_set(&a16, 1UL << 10);
318 rte_atomic32_set(&a32, 1UL << 10);
319 rte_atomic64_set(&a64, 1ULL << 33);
321 printf("usual inc/dec/add/sub functions\n");
323 rte_eal_mp_remote_launch(test_atomic_usual, NULL, SKIP_MASTER);
324 rte_atomic32_set(&synchro, 1);
325 rte_eal_mp_wait_lcore();
326 rte_atomic32_set(&synchro, 0);
328 if (rte_atomic16_read(&a16) != 1UL << 10) {
329 printf("Atomic16 usual functions failed\n");
333 if (rte_atomic32_read(&a32) != 1UL << 10) {
334 printf("Atomic32 usual functions failed\n");
338 if (rte_atomic64_read(&a64) != 1ULL << 33) {
339 printf("Atomic64 usual functions failed\n");
343 printf("test and set\n");
345 rte_atomic64_set(&a64, 0);
346 rte_atomic32_set(&a32, 0);
347 rte_atomic16_set(&a16, 0);
348 rte_atomic64_set(&count, 0);
349 rte_eal_mp_remote_launch(test_atomic_tas, NULL, SKIP_MASTER);
350 rte_atomic32_set(&synchro, 1);
351 rte_eal_mp_wait_lcore();
352 rte_atomic32_set(&synchro, 0);
354 if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) {
355 printf("Atomic test and set failed\n");
359 printf("add/sub and return\n");
361 rte_atomic64_set(&a64, 0);
362 rte_atomic32_set(&a32, 0);
363 rte_atomic16_set(&a16, 0);
364 rte_atomic64_set(&count, 0);
365 rte_eal_mp_remote_launch(test_atomic_addsub_and_return, NULL,
367 rte_atomic32_set(&synchro, 1);
368 rte_eal_mp_wait_lcore();
369 rte_atomic32_set(&synchro, 0);
371 if (rte_atomic64_read(&count) != 0) {
372 printf("Atomic add/sub+return failed\n");
377 * Set a64, a32 and a16 with the same value of minus "number of slave
378 * lcores", launch all slave lcores to atomically increase by one and
379 * test them respectively.
380 * Each lcore should have only one chance to increase a64 by one and
381 * then check if it is equal to 0, but there should be only one lcore
382 * that finds that it is 0. It is similar for a32 and a16.
383 * Then a variable of "count", initialized to zero, is increased by
384 * one if a64, a32 or a16 is 0 after being increased and tested
386 * We can check if "count" is finally equal to 3 to see if all slave
387 * lcores performed "atomic inc and test" right.
389 printf("inc and test\n");
391 rte_atomic64_clear(&a64);
392 rte_atomic32_clear(&a32);
393 rte_atomic16_clear(&a16);
394 rte_atomic32_clear(&synchro);
395 rte_atomic64_clear(&count);
397 rte_atomic64_set(&a64, (int64_t)(1 - (int64_t)rte_lcore_count()));
398 rte_atomic32_set(&a32, (int32_t)(1 - (int32_t)rte_lcore_count()));
399 rte_atomic16_set(&a16, (int16_t)(1 - (int16_t)rte_lcore_count()));
400 rte_eal_mp_remote_launch(test_atomic_inc_and_test, NULL, SKIP_MASTER);
401 rte_atomic32_set(&synchro, 1);
402 rte_eal_mp_wait_lcore();
403 rte_atomic32_clear(&synchro);
405 if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) {
406 printf("Atomic inc and test failed %d\n", (int)count.cnt);
411 * Same as above, but this time we set the values to "number of slave
412 * lcores", and decrement instead of increment.
414 printf("dec and test\n");
416 rte_atomic32_clear(&synchro);
417 rte_atomic64_clear(&count);
419 rte_atomic64_set(&a64, (int64_t)(rte_lcore_count() - 1));
420 rte_atomic32_set(&a32, (int32_t)(rte_lcore_count() - 1));
421 rte_atomic16_set(&a16, (int16_t)(rte_lcore_count() - 1));
422 rte_eal_mp_remote_launch(test_atomic_dec_and_test, NULL, SKIP_MASTER);
423 rte_atomic32_set(&synchro, 1);
424 rte_eal_mp_wait_lcore();
425 rte_atomic32_clear(&synchro);
427 if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) {
428 printf("Atomic dec and test failed\n");
432 #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
434 * This case tests the functionality of rte_atomic128_cmp_exchange
435 * API. It calls rte_atomic128_cmp_exchange with four kinds of memory
436 * models successively on each slave core. Once each 128-bit atomic
437 * compare and swap operation is successful, it updates the global
438 * 128-bit counter by 2 for the first 64-bit and 1 for the second
439 * 64-bit. Each slave core iterates this test N times.
440 * At the end of test, verify whether the first 64-bits of the 128-bit
441 * counter and the second 64bits is differ by the total iterations. If
442 * it is, the test passes.
444 printf("128-bit compare and swap test\n");
445 uint64_t iterations = 0;
447 rte_atomic32_clear(&synchro);
451 rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
453 rte_atomic32_set(&synchro, 1);
454 rte_eal_mp_wait_lcore();
455 rte_atomic32_clear(&synchro);
457 iterations = count128.val[0] - count128.val[1];
458 if (iterations != 4*N*(rte_lcore_count()-1)) {
459 printf("128-bit compare and swap failed\n");
467 REGISTER_TEST_COMMAND(atomic_autotest, test_atomic);