1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
8 #include <rte_cycles.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
19 #define RTE_RWTEST_FAIL 0
21 #define TOTAL_ENTRY (5*1024*1024)
22 #define TOTAL_INSERT (4.5*1024*1024)
25 unsigned int core_cnt[NUM_TEST] = {2, 4, 8};
27 unsigned int slave_core_ids[RTE_MAX_LCORE];
30 uint32_t single_write;
31 uint32_t read_only[NUM_TEST];
32 uint32_t write_only[NUM_TEST];
33 uint32_t read_write_r[NUM_TEST];
34 uint32_t read_write_w[NUM_TEST];
37 static struct perf htm_results, non_htm_results;
43 uint32_t rounded_tot_insert;
47 static rte_atomic64_t gcycles;
48 static rte_atomic64_t ginsertions;
50 static rte_atomic64_t gread_cycles;
51 static rte_atomic64_t gwrite_cycles;
53 static rte_atomic64_t greads;
54 static rte_atomic64_t gwrites;
57 test_hash_readwrite_worker(__attribute__((unused)) void *arg)
60 uint32_t lcore_id = rte_lcore_id();
61 uint64_t begin, cycles;
64 ret = rte_malloc(NULL, sizeof(int) *
65 tbl_rw_test_param.num_insert, 0);
66 for (i = 0; i < rte_lcore_count(); i++) {
67 if (slave_core_ids[i] == lcore_id)
70 offset = tbl_rw_test_param.num_insert * i;
72 printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n",
73 lcore_id, tbl_rw_test_param.num_insert,
74 offset, offset + tbl_rw_test_param.num_insert - 1);
76 begin = rte_rdtsc_precise();
78 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
80 if (rte_hash_lookup(tbl_rw_test_param.h,
81 tbl_rw_test_param.keys + i) > 0)
84 ret[i - offset] = rte_hash_add_key(tbl_rw_test_param.h,
85 tbl_rw_test_param.keys + i);
86 if (ret[i - offset] < 0)
89 /* lookup a random key */
90 uint32_t rand = rte_rand() % (i + 1 - offset);
92 if (rte_hash_lookup(tbl_rw_test_param.h,
93 tbl_rw_test_param.keys + rand) != ret[rand])
97 if (rte_hash_del_key(tbl_rw_test_param.h,
98 tbl_rw_test_param.keys + rand) != ret[rand])
101 ret[rand] = rte_hash_add_key(tbl_rw_test_param.h,
102 tbl_rw_test_param.keys + rand);
106 if (rte_hash_lookup(tbl_rw_test_param.h,
107 tbl_rw_test_param.keys + rand) != ret[rand])
111 cycles = rte_rdtsc_precise() - begin;
112 rte_atomic64_add(&gcycles, cycles);
113 rte_atomic64_add(&ginsertions, i - offset);
115 for (; i < offset + tbl_rw_test_param.num_insert; i++)
116 tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL;
123 init_params(int use_htm, int use_jhash)
127 uint32_t *keys = NULL;
128 uint32_t *found = NULL;
129 struct rte_hash *handle;
131 struct rte_hash_parameters hash_params = {
132 .entries = TOTAL_ENTRY,
133 .key_len = sizeof(uint32_t),
134 .hash_func_init_val = 0,
135 .socket_id = rte_socket_id(),
138 hash_params.hash_func = rte_jhash;
140 hash_params.hash_func = rte_hash_crc;
143 hash_params.extra_flag =
144 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
145 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
146 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
148 hash_params.extra_flag =
149 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
150 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
152 hash_params.name = "tests";
154 handle = rte_hash_create(&hash_params);
155 if (handle == NULL) {
156 printf("hash creation failed");
160 tbl_rw_test_param.h = handle;
161 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
164 printf("RTE_MALLOC failed\n");
168 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
170 printf("RTE_ZMALLOC failed\n");
174 tbl_rw_test_param.keys = keys;
175 tbl_rw_test_param.found = found;
177 for (i = 0; i < TOTAL_ENTRY; i++)
184 rte_hash_free(handle);
190 test_hash_readwrite_functional(int use_htm)
193 const void *next_key;
197 uint32_t duplicated_keys = 0;
198 uint32_t lost_keys = 0;
200 int slave_cnt = rte_lcore_count() - 1;
202 rte_atomic64_init(&gcycles);
203 rte_atomic64_clear(&gcycles);
205 rte_atomic64_init(&ginsertions);
206 rte_atomic64_clear(&ginsertions);
208 if (init_params(use_htm, use_jhash) != 0)
211 tbl_rw_test_param.num_insert =
212 TOTAL_INSERT / slave_cnt;
214 tbl_rw_test_param.rounded_tot_insert =
215 tbl_rw_test_param.num_insert
218 printf("++++++++Start function tests:+++++++++\n");
220 /* Fire all threads. */
221 rte_eal_mp_remote_launch(test_hash_readwrite_worker,
223 rte_eal_mp_wait_lcore();
225 while (rte_hash_iterate(tbl_rw_test_param.h, &next_key,
226 &next_data, &iter) >= 0) {
227 /* Search for the key in the list of keys added .*/
228 i = *(const uint32_t *)next_key;
229 tbl_rw_test_param.found[i]++;
232 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
233 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
234 if (tbl_rw_test_param.found[i] > 1) {
238 if (tbl_rw_test_param.found[i] == 0) {
240 printf("key %d is lost\n", i);
246 if (duplicated_keys > 0) {
247 printf("%d key duplicated\n", duplicated_keys);
252 printf("%d key lost\n", lost_keys);
256 printf("No key corrupted during read-write test.\n");
258 unsigned long long int cycles_per_insertion =
259 rte_atomic64_read(&gcycles) /
260 rte_atomic64_read(&ginsertions);
262 printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion);
264 rte_free(tbl_rw_test_param.found);
265 rte_free(tbl_rw_test_param.keys);
266 rte_hash_free(tbl_rw_test_param.h);
267 printf("+++++++++Complete function tests+++++++++\n");
271 rte_free(tbl_rw_test_param.found);
272 rte_free(tbl_rw_test_param.keys);
273 rte_hash_free(tbl_rw_test_param.h);
279 test_rw_reader(void *arg)
282 uint64_t begin, cycles;
283 uint64_t read_cnt = (uint64_t)((uintptr_t)arg);
285 begin = rte_rdtsc_precise();
286 for (i = 0; i < read_cnt; i++) {
288 rte_hash_lookup_data(tbl_rw_test_param.h,
289 tbl_rw_test_param.keys + i,
291 if (i != (uint64_t)(uintptr_t)data) {
292 printf("lookup find wrong value %"PRIu64","
294 (uint64_t)(uintptr_t)data);
299 cycles = rte_rdtsc_precise() - begin;
300 rte_atomic64_add(&gread_cycles, cycles);
301 rte_atomic64_add(&greads, i);
306 test_rw_writer(void *arg)
309 uint32_t lcore_id = rte_lcore_id();
310 uint64_t begin, cycles;
312 uint64_t start_coreid = (uint64_t)(uintptr_t)arg;
315 for (i = 0; i < rte_lcore_count(); i++) {
316 if (slave_core_ids[i] == lcore_id)
320 offset = TOTAL_INSERT / 2 + (i - (start_coreid)) *
321 tbl_rw_test_param.num_insert;
322 begin = rte_rdtsc_precise();
323 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
324 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
325 tbl_rw_test_param.keys + i,
326 (void *)((uintptr_t)i));
328 printf("writer failed %"PRIu64"\n", i);
333 cycles = rte_rdtsc_precise() - begin;
334 rte_atomic64_add(&gwrite_cycles, cycles);
335 rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert);
340 test_hash_readwrite_perf(struct perf *perf_results, int use_htm,
346 uint64_t i, read_cnt;
348 const void *next_key;
353 uint32_t duplicated_keys = 0;
354 uint32_t lost_keys = 0;
356 uint64_t start = 0, end = 0;
358 rte_atomic64_init(&greads);
359 rte_atomic64_init(&gwrites);
360 rte_atomic64_clear(&gwrites);
361 rte_atomic64_clear(&greads);
363 rte_atomic64_init(&gread_cycles);
364 rte_atomic64_clear(&gread_cycles);
365 rte_atomic64_init(&gwrite_cycles);
366 rte_atomic64_clear(&gwrite_cycles);
368 if (init_params(use_htm, use_jhash) != 0)
372 * Do a readers finish faster or writers finish faster test.
373 * When readers finish faster, we timing the readers, and when writers
374 * finish faster, we timing the writers.
375 * Divided by 10 or 2 is just experimental values to vary the workload
379 printf("++++++Start perf test: reader++++++++\n");
380 read_cnt = TOTAL_INSERT / 10;
382 printf("++++++Start perf test: writer++++++++\n");
383 read_cnt = TOTAL_INSERT / 2;
386 /* We first test single thread performance */
387 start = rte_rdtsc_precise();
388 /* Insert half of the keys */
389 for (i = 0; i < TOTAL_INSERT / 2; i++) {
390 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
391 tbl_rw_test_param.keys + i,
392 (void *)((uintptr_t)i));
394 printf("Failed to insert half of keys\n");
398 end = rte_rdtsc_precise() - start;
399 perf_results->single_write = end / i;
401 start = rte_rdtsc_precise();
403 for (i = 0; i < read_cnt; i++) {
405 rte_hash_lookup_data(tbl_rw_test_param.h,
406 tbl_rw_test_param.keys + i,
408 if (i != (uint64_t)(uintptr_t)data) {
409 printf("lookup find wrong value"
410 " %"PRIu64",%"PRIu64"\n", i,
411 (uint64_t)(uintptr_t)data);
415 end = rte_rdtsc_precise() - start;
416 perf_results->single_read = end / i;
418 for (n = 0; n < NUM_TEST; n++) {
419 unsigned int tot_slave_lcore = rte_lcore_count() - 1;
420 if (tot_slave_lcore < core_cnt[n] * 2)
423 rte_atomic64_clear(&greads);
424 rte_atomic64_clear(&gread_cycles);
425 rte_atomic64_clear(&gwrites);
426 rte_atomic64_clear(&gwrite_cycles);
428 rte_hash_reset(tbl_rw_test_param.h);
430 tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n];
431 tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 +
432 tbl_rw_test_param.num_insert *
435 for (i = 0; i < TOTAL_INSERT / 2; i++) {
436 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
437 tbl_rw_test_param.keys + i,
438 (void *)((uintptr_t)i));
440 printf("Failed to insert half of keys\n");
445 /* Then test multiple thread case but only all reads or
449 /* Test only reader cases */
450 for (i = 0; i < core_cnt[n]; i++)
451 rte_eal_remote_launch(test_rw_reader,
452 (void *)(uintptr_t)read_cnt,
455 rte_eal_mp_wait_lcore();
458 /* Test only writer cases */
459 for (; i < core_cnt[n] * 2; i++)
460 rte_eal_remote_launch(test_rw_writer,
461 (void *)((uintptr_t)start_coreid),
464 rte_eal_mp_wait_lcore();
467 unsigned long long int cycles_per_insertion =
468 rte_atomic64_read(&gread_cycles) /
469 rte_atomic64_read(&greads);
470 perf_results->read_only[n] = cycles_per_insertion;
471 printf("Reader only: cycles per lookup: %llu\n",
472 cycles_per_insertion);
476 unsigned long long int cycles_per_insertion =
477 rte_atomic64_read(&gwrite_cycles) /
478 rte_atomic64_read(&gwrites);
479 perf_results->write_only[n] = cycles_per_insertion;
480 printf("Writer only: cycles per writes: %llu\n",
481 cycles_per_insertion);
484 rte_atomic64_clear(&greads);
485 rte_atomic64_clear(&gread_cycles);
486 rte_atomic64_clear(&gwrites);
487 rte_atomic64_clear(&gwrite_cycles);
489 rte_hash_reset(tbl_rw_test_param.h);
491 for (i = 0; i < TOTAL_INSERT / 2; i++) {
492 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
493 tbl_rw_test_param.keys + i,
494 (void *)((uintptr_t)i));
496 printf("Failed to insert half of keys\n");
501 start_coreid = core_cnt[n];
504 for (i = core_cnt[n]; i < core_cnt[n] * 2; i++)
505 rte_eal_remote_launch(test_rw_writer,
506 (void *)((uintptr_t)start_coreid),
508 for (i = 0; i < core_cnt[n]; i++)
509 rte_eal_remote_launch(test_rw_reader,
510 (void *)(uintptr_t)read_cnt,
513 for (i = 0; i < core_cnt[n]; i++)
514 rte_eal_remote_launch(test_rw_reader,
515 (void *)(uintptr_t)read_cnt,
517 for (; i < core_cnt[n] * 2; i++)
518 rte_eal_remote_launch(test_rw_writer,
519 (void *)((uintptr_t)start_coreid),
523 rte_eal_mp_wait_lcore();
525 while (rte_hash_iterate(tbl_rw_test_param.h,
526 &next_key, &next_data, &iter) >= 0) {
527 /* Search for the key in the list of keys added .*/
528 i = *(const uint32_t *)next_key;
529 tbl_rw_test_param.found[i]++;
532 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
533 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
534 if (tbl_rw_test_param.found[i] > 1) {
538 if (tbl_rw_test_param.found[i] == 0) {
540 printf("key %"PRIu64" is lost\n", i);
546 if (duplicated_keys > 0) {
547 printf("%d key duplicated\n", duplicated_keys);
552 printf("%d key lost\n", lost_keys);
556 printf("No key corrupted during read-write test.\n");
559 unsigned long long int cycles_per_insertion =
560 rte_atomic64_read(&gread_cycles) /
561 rte_atomic64_read(&greads);
562 perf_results->read_write_r[n] = cycles_per_insertion;
563 printf("Read-write cycles per lookup: %llu\n",
564 cycles_per_insertion);
568 unsigned long long int cycles_per_insertion =
569 rte_atomic64_read(&gwrite_cycles) /
570 rte_atomic64_read(&gwrites);
571 perf_results->read_write_w[n] = cycles_per_insertion;
572 printf("Read-write cycles per writes: %llu\n",
573 cycles_per_insertion);
578 rte_free(tbl_rw_test_param.found);
579 rte_free(tbl_rw_test_param.keys);
580 rte_hash_free(tbl_rw_test_param.h);
584 rte_free(tbl_rw_test_param.found);
585 rte_free(tbl_rw_test_param.keys);
586 rte_hash_free(tbl_rw_test_param.h);
593 test_hash_readwrite_main(void)
596 * Variables used to choose different tests.
597 * use_htm indicates if hardware transactional memory should be used.
598 * reader_faster indicates if the reader threads should finish earlier
599 * than writer threads. This is to timing either reader threads or
600 * writer threads for performance numbers.
602 int use_htm, reader_faster;
603 unsigned int i = 0, core_id = 0;
605 if (rte_lcore_count() <= 2) {
606 printf("More than two lcores are required "
607 "to do read write test\n");
611 RTE_LCORE_FOREACH_SLAVE(core_id) {
612 slave_core_ids[i] = core_id;
616 setlocale(LC_NUMERIC, "");
618 if (rte_tm_supported()) {
619 printf("Hardware transactional memory (lock elision) "
622 printf("Test read-write with Hardware transactional memory\n");
625 if (test_hash_readwrite_functional(use_htm) < 0)
629 if (test_hash_readwrite_perf(&htm_results, use_htm,
634 if (test_hash_readwrite_perf(&htm_results, use_htm,
638 printf("Hardware transactional memory (lock elision) "
639 "is NOT supported\n");
642 printf("Test read-write without Hardware transactional memory\n");
644 if (test_hash_readwrite_functional(use_htm) < 0)
647 if (test_hash_readwrite_perf(&non_htm_results, use_htm,
651 if (test_hash_readwrite_perf(&non_htm_results, use_htm,
655 printf("Results summary:\n");
657 printf("single read: %u\n", htm_results.single_read);
658 printf("single write: %u\n", htm_results.single_write);
659 for (i = 0; i < NUM_TEST; i++) {
660 printf("core_cnt: %u\n", core_cnt[i]);
662 printf("read only: %u\n", htm_results.read_only[i]);
663 printf("write only: %u\n", htm_results.write_only[i]);
664 printf("read-write read: %u\n", htm_results.read_write_r[i]);
665 printf("read-write write: %u\n", htm_results.read_write_w[i]);
667 printf("non HTM:\n");
668 printf("read only: %u\n", non_htm_results.read_only[i]);
669 printf("write only: %u\n", non_htm_results.write_only[i]);
670 printf("read-write read: %u\n",
671 non_htm_results.read_write_r[i]);
672 printf("read-write write: %u\n",
673 non_htm_results.read_write_w[i]);
679 REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main);