1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Arm Limited
8 #include <rte_cycles.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
23 #define BULK_LOOKUP_SIZE 32
25 #define RUN_WITH_HTM_DISABLED 0
27 #if (RUN_WITH_HTM_DISABLED)
29 #define TOTAL_ENTRY (5*1024)
30 #define TOTAL_INSERT (5*1024)
34 #define TOTAL_ENTRY (4*1024*1024)
35 #define TOTAL_INSERT (4*1024*1024)
40 #define READ_PASS_NO_KEY_SHIFTS 2
41 #define READ_PASS_SHIFT_PATH 4
42 #define READ_PASS_NON_SHIFT_PATH 8
43 #define BULK_LOOKUP 16
44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
46 #define WRITE_NO_KEY_SHIFT 0
47 #define WRITE_KEY_SHIFT 1
48 #define WRITE_EXT_BKT 2
51 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
54 uint32_t w_no_ks_r_hit[2][NUM_TEST];
55 uint32_t w_no_ks_r_miss[2][NUM_TEST];
56 uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
57 uint32_t w_ks_r_hit_sp[2][NUM_TEST];
58 uint32_t w_ks_r_miss[2][NUM_TEST];
59 uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
60 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
69 uint32_t *keys_absent;
70 uint32_t *keys_shift_path;
71 uint32_t *keys_non_shift_path;
72 uint32_t *keys_ext_bkt;
73 uint32_t *keys_ks_extbkt;
74 uint32_t count_keys_no_ks;
75 uint32_t count_keys_ks;
76 uint32_t count_keys_absent;
77 uint32_t count_keys_shift_path;
78 uint32_t count_keys_non_shift_path;
79 uint32_t count_keys_extbkt;
80 uint32_t count_keys_ks_extbkt;
81 uint32_t single_insert;
85 static uint64_t gread_cycles;
86 static uint64_t greads;
88 static volatile uint8_t writer_done;
90 static uint16_t enabled_core_ids[RTE_MAX_LCORE];
92 static uint8_t *scanned_bkts;
94 static inline uint16_t
95 get_short_sig(const hash_sig_t hash)
100 static inline uint32_t
101 get_prim_bucket_index(__rte_unused const struct rte_hash *h,
102 const hash_sig_t hash)
104 uint32_t num_buckets;
105 uint32_t bucket_bitmask;
106 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
107 bucket_bitmask = num_buckets - 1;
108 return hash & bucket_bitmask;
111 static inline uint32_t
112 get_alt_bucket_index(__rte_unused const struct rte_hash *h,
113 uint32_t cur_bkt_idx, uint16_t sig)
115 uint32_t num_buckets;
116 uint32_t bucket_bitmask;
117 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
118 bucket_bitmask = num_buckets - 1;
119 return (cur_bkt_idx ^ sig) & bucket_bitmask;
124 get_enabled_cores_list(void)
128 uint32_t max_cores = rte_lcore_count();
129 RTE_LCORE_FOREACH(core_id) {
130 enabled_core_ids[i] = core_id;
134 if (i != max_cores) {
135 printf("Number of enabled cores in list is different from "
136 "number given by rte_lcore_count()\n");
143 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
145 struct rte_hash *handle;
147 struct rte_hash_parameters hash_params = {
148 .entries = TOTAL_ENTRY,
149 .key_len = sizeof(uint32_t),
150 .hash_func_init_val = 0,
151 .socket_id = rte_socket_id(),
155 hash_params.hash_func = rte_jhash;
157 hash_params.hash_func = rte_hash_crc;
160 hash_params.extra_flag =
161 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
162 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
164 hash_params.extra_flag =
165 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
166 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
167 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
169 hash_params.extra_flag =
170 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
171 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
174 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
176 hash_params.name = "tests";
178 handle = rte_hash_create(&hash_params);
179 if (handle == NULL) {
180 printf("hash creation failed");
184 tbl_rwc_test_param.h = handle;
189 check_bucket(uint32_t bkt_idx, uint32_t key)
195 const void *next_key;
198 /* Temporary bucket to hold the keys */
199 uint32_t keys_in_bkt[8];
203 while (rte_hash_iterate(tbl_rwc_test_param.h,
204 &next_key, &next_data, &iter) >= 0) {
206 /* Check for duplicate entries */
207 if (*(const uint32_t *)next_key == key)
210 /* Identify if there is any free entry in the bucket */
211 diff = iter - prev_iter;
216 keys_in_bkt[count] = *(const uint32_t *)next_key;
219 /* All entries in the bucket are occupied */
223 * Check if bucket was not scanned before, to avoid
226 if (scanned_bkts[bkt_idx] == 0) {
228 * Since this bucket (pointed to by bkt_idx) is
229 * full, it is likely that key(s) in this
230 * bucket will be on the shift path, when
231 * collision occurs. Thus, add it to
234 memcpy(tbl_rwc_test_param.keys_shift_path +
235 tbl_rwc_test_param.count_keys_shift_path
237 tbl_rwc_test_param.count_keys_shift_path += 8;
238 scanned_bkts[bkt_idx] = 1;
249 uint32_t *keys = NULL;
250 uint32_t *keys_no_ks = NULL;
251 uint32_t *keys_ks = NULL;
252 uint32_t *keys_absent = NULL;
253 uint32_t *keys_non_shift_path = NULL;
254 uint32_t *keys_ext_bkt = NULL;
255 uint32_t *keys_ks_extbkt = NULL;
256 uint32_t *found = NULL;
257 uint32_t count_keys_no_ks = 0;
258 uint32_t count_keys_ks = 0;
259 uint32_t count_keys_extbkt = 0;
262 if (init_params(0, 0, 0, 0) != 0)
266 * keys will consist of a) keys whose addition to the hash table
267 * will result in shifting of the existing keys to their alternate
268 * locations b) keys whose addition to the hash table will not result
269 * in shifting of the existing keys.
271 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
273 printf("RTE_MALLOC failed\n");
278 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that
279 * will NOT result in shifting of the existing keys to their alternate
280 * locations. Roughly around 900K keys.
282 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
283 if (keys_no_ks == NULL) {
284 printf("RTE_MALLOC failed\n");
289 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
290 * result in shifting of the existing keys to their alternate locations.
291 * Roughly around 146K keys. There might be repeating keys. More code is
292 * required to filter out these keys which will complicate the test case
294 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
295 if (keys_ks == NULL) {
296 printf("RTE_MALLOC failed\n");
300 /* Used to identify keys not inserted in the hash table */
301 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
303 printf("RTE_MALLOC failed\n");
308 * This consist of keys not inserted to the hash table.
309 * Used to test perf of lookup on keys that do not exist in the table.
311 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
312 if (keys_absent == NULL) {
313 printf("RTE_MALLOC failed\n");
318 * This consist of keys which are likely to be on the shift
319 * path (i.e. being moved to alternate location), when collision occurs
320 * on addition of a key to an already full primary bucket.
321 * Used to test perf of lookup on keys that are on the shift path.
323 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
325 if (tbl_rwc_test_param.keys_shift_path == NULL) {
326 printf("RTE_MALLOC failed\n");
331 * This consist of keys which are never on the shift
332 * path (i.e. being moved to alternate location), when collision occurs
333 * on addition of a key to an already full primary bucket.
334 * Used to test perf of lookup on keys that are not on the shift path.
336 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
338 if (keys_non_shift_path == NULL) {
339 printf("RTE_MALLOC failed\n");
344 * This consist of keys which will be stored in extended buckets
346 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
347 if (keys_ext_bkt == NULL) {
348 printf("RTE_MALLOC failed\n");
353 * This consist of keys which when deleted causes shifting of keys
354 * in extended buckets to respective secondary buckets
356 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
357 if (keys_ks_extbkt == NULL) {
358 printf("RTE_MALLOC failed\n");
363 uint32_t prim_bucket_idx;
364 uint32_t sec_bucket_idx;
366 uint32_t num_buckets;
367 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
371 * Used to mark bkts in which at least one key was shifted to its
374 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
375 if (scanned_bkts == NULL) {
376 printf("RTE_MALLOC failed\n");
380 tbl_rwc_test_param.keys = keys;
381 tbl_rwc_test_param.keys_no_ks = keys_no_ks;
382 tbl_rwc_test_param.keys_ks = keys_ks;
383 tbl_rwc_test_param.keys_absent = keys_absent;
384 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
385 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
386 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
387 /* Generate keys by adding previous two keys, neglect overflow */
388 printf("Generating keys...\n");
391 for (i = 2; i < TOTAL_INSERT; i++)
392 keys[i] = keys[i-1] + keys[i-2];
394 /* Segregate keys into keys_no_ks and keys_ks */
395 for (i = 0; i < TOTAL_INSERT; i++) {
396 /* Check if primary bucket has space.*/
397 sig = rte_hash_hash(tbl_rwc_test_param.h,
398 tbl_rwc_test_param.keys+i);
399 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
401 ret = check_bucket(prim_bucket_idx, keys[i]);
404 * Primary bucket is full, this key will result in
405 * shifting of the keys to their alternate locations.
407 keys_ks[count_keys_ks] = keys[i];
409 } else if (ret == 0) {
411 * Primary bucket has space, this key will not result in
412 * shifting of the keys. Hence, add key to the table.
414 ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
416 (void *)((uintptr_t)i));
418 printf("writer failed %"PRIu32"\n", i);
421 keys_no_ks[count_keys_no_ks] = keys[i];
426 for (i = 0; i < count_keys_no_ks; i++) {
428 * Identify keys in keys_no_ks with value less than
429 * 4M (HTM enabled) OR 5K (HTM disabled)
431 if (keys_no_ks[i] < TOTAL_INSERT)
432 found[keys_no_ks[i]]++;
435 for (i = 0; i < count_keys_ks; i++) {
437 * Identify keys in keys_ks with value less than
438 * 4M (HTM enabled) OR 5K (HTM disabled)
440 if (keys_ks[i] < TOTAL_INSERT)
444 uint32_t count_keys_absent = 0;
445 for (i = 0; i < TOTAL_INSERT; i++) {
447 * Identify missing keys between 0 and
448 * 4M (HTM enabled) OR 5K (HTM disabled)
451 keys_absent[count_keys_absent++] = i;
454 /* Find keys that will not be on the shift path */
456 const void *next_key;
459 for (i = 0; i < num_buckets; i++) {
460 /* Check bucket for no keys shifted to alternate locations */
461 if (scanned_bkts[i] == 0) {
463 while (rte_hash_iterate(tbl_rwc_test_param.h,
464 &next_key, &next_data, &iter) >= 0) {
466 /* Check if key belongs to the current bucket */
468 keys_non_shift_path[count++]
469 = *(const uint32_t *)next_key;
476 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
477 tbl_rwc_test_param.count_keys_ks = count_keys_ks;
478 tbl_rwc_test_param.count_keys_absent = count_keys_absent;
479 tbl_rwc_test_param.count_keys_non_shift_path = count;
481 memset(scanned_bkts, 0, num_buckets);
483 /* Find keys that will be in extended buckets */
484 for (i = 0; i < count_keys_ks; i++) {
485 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
487 /* Key will be added to ext bkt */
488 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
489 /* Sec bkt to be added to keys_ks_extbkt */
490 sig = rte_hash_hash(tbl_rwc_test_param.h,
491 tbl_rwc_test_param.keys_ks + i);
492 prim_bucket_idx = get_prim_bucket_index(
493 tbl_rwc_test_param.h, sig);
494 short_sig = get_short_sig(sig);
495 sec_bucket_idx = get_alt_bucket_index(
496 tbl_rwc_test_param.h,
497 prim_bucket_idx, short_sig);
498 if (scanned_bkts[sec_bucket_idx] == 0)
499 scanned_bkts[sec_bucket_idx] = 1;
503 /* Find keys that will shift keys in ext bucket*/
504 for (i = 0; i < num_buckets; i++) {
505 if (scanned_bkts[i] == 1) {
507 while (rte_hash_iterate(tbl_rwc_test_param.h,
508 &next_key, &next_data, &iter) >= 0) {
509 /* Check if key belongs to the current bucket */
511 keys_ks_extbkt[count++]
512 = *(const uint32_t *)next_key;
519 tbl_rwc_test_param.count_keys_ks_extbkt = count;
520 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
522 printf("\nCount of keys NOT causing shifting of existing keys to "
523 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
524 printf("\nCount of keys causing shifting of existing keys to alternate "
525 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
526 printf("Count of absent keys that will never be added to the hash "
527 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
528 printf("Count of keys likely to be on the shift path: %d\n\n",
529 tbl_rwc_test_param.count_keys_shift_path);
530 printf("Count of keys not likely to be on the shift path: %d\n\n",
531 tbl_rwc_test_param.count_keys_non_shift_path);
532 printf("Count of keys in extended buckets: %d\n\n",
533 tbl_rwc_test_param.count_keys_extbkt);
534 printf("Count of keys shifting keys in ext buckets: %d\n\n",
535 tbl_rwc_test_param.count_keys_ks_extbkt);
538 rte_free(scanned_bkts);
539 rte_hash_free(tbl_rwc_test_param.h);
544 rte_free(keys_no_ks);
546 rte_free(keys_absent);
548 rte_free(tbl_rwc_test_param.keys_shift_path);
549 rte_free(keys_non_shift_path);
550 rte_free(keys_ext_bkt);
551 rte_free(keys_ks_extbkt);
552 rte_free(scanned_bkts);
553 rte_hash_free(tbl_rwc_test_param.h);
558 test_rwc_reader(__rte_unused void *arg)
562 uint64_t begin, cycles;
563 uint32_t loop_cnt = 0;
564 uint8_t read_type = (uint8_t)((uintptr_t)arg);
568 int32_t pos[BULK_LOOKUP_SIZE];
569 void *temp_a[BULK_LOOKUP_SIZE];
571 if (read_type & READ_FAIL) {
572 keys = tbl_rwc_test_param.keys_absent;
573 read_cnt = tbl_rwc_test_param.count_keys_absent;
574 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
575 keys = tbl_rwc_test_param.keys_no_ks;
576 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
577 } else if (read_type & READ_PASS_SHIFT_PATH) {
578 keys = tbl_rwc_test_param.keys_shift_path;
579 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
580 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
581 keys = tbl_rwc_test_param.keys_ext_bkt;
582 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
584 keys = tbl_rwc_test_param.keys_non_shift_path;
585 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
588 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
590 begin = rte_rdtsc_precise();
592 if (read_type & BULK_LOOKUP) {
593 for (i = 0; i < (read_cnt - extra_keys);
594 i += BULK_LOOKUP_SIZE) {
595 /* Array of pointer to the list of keys */
596 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
597 temp_a[j] = keys + i + j;
598 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
601 BULK_LOOKUP_SIZE, pos);
602 /* Validate lookup result */
603 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
604 if ((read_type & READ_FAIL &&
605 pos[j] != -ENOENT) ||
606 (!(read_type & READ_FAIL) &&
607 pos[j] == -ENOENT)) {
608 printf("lookup failed!"
614 for (j = 0; j < extra_keys; j++)
615 temp_a[j] = keys + i + j;
617 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
621 for (j = 0; j < extra_keys; j++)
622 if ((read_type & READ_FAIL &&
623 pos[j] != -ENOENT) ||
624 (!(read_type & READ_FAIL) &&
625 pos[j] == -ENOENT)) {
626 printf("lookup failed! %"PRIu32"\n",
631 for (i = 0; i < read_cnt; i++) {
632 ret = rte_hash_lookup
633 (tbl_rwc_test_param.h, keys + i);
634 if (((read_type & READ_FAIL) &&
636 (!(read_type & READ_FAIL) &&
638 printf("lookup failed! %"PRIu32"\n",
645 } while (!writer_done);
647 cycles = rte_rdtsc_precise() - begin;
648 __atomic_fetch_add(&gread_cycles, cycles, __ATOMIC_RELAXED);
649 __atomic_fetch_add(&greads, read_cnt*loop_cnt, __ATOMIC_RELAXED);
654 write_keys(uint8_t write_type)
658 uint32_t key_cnt = 0;
660 if (write_type == WRITE_KEY_SHIFT) {
661 key_cnt = tbl_rwc_test_param.count_keys_ks;
662 keys = tbl_rwc_test_param.keys_ks;
663 } else if (write_type == WRITE_NO_KEY_SHIFT) {
664 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
665 keys = tbl_rwc_test_param.keys_no_ks;
666 } else if (write_type == WRITE_EXT_BKT) {
667 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
668 keys = tbl_rwc_test_param.keys_ext_bkt;
670 for (i = 0; i < key_cnt; i++) {
671 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
672 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
673 printf("writer failed %"PRIu32"\n", i);
681 test_rwc_multi_writer(__rte_unused void *arg)
684 uint32_t pos_core = (uint32_t)((uintptr_t)arg);
685 offset = pos_core * tbl_rwc_test_param.single_insert;
686 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
687 rte_hash_add_key(tbl_rwc_test_param.h,
688 tbl_rwc_test_param.keys_ks + i);
694 * Reader(s) lookup keys present in the table.
697 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
698 int htm, int ext_bkt)
703 uint8_t write_type = WRITE_NO_KEY_SHIFT;
704 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
706 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
708 printf("\nTest: Hash add - no key-shifts, read - hit\n");
709 for (m = 0; m < 2; m++) {
711 printf("\n** With bulk-lookup **\n");
712 read_type |= BULK_LOOKUP;
714 for (n = 0; n < NUM_TEST; n++) {
715 unsigned int tot_lcore = rte_lcore_count();
716 if (tot_lcore < rwc_core_cnt[n] + 1)
719 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
721 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
722 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
724 rte_hash_reset(tbl_rwc_test_param.h);
726 if (write_keys(write_type) < 0)
729 for (i = 1; i <= rwc_core_cnt[n]; i++)
730 rte_eal_remote_launch(test_rwc_reader,
731 (void *)(uintptr_t)read_type,
732 enabled_core_ids[i]);
734 for (i = 1; i <= rwc_core_cnt[n]; i++)
735 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
738 unsigned long long cycles_per_lookup =
739 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
740 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
741 rwc_perf_results->w_no_ks_r_hit[m][n]
743 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
748 rte_hash_free(tbl_rwc_test_param.h);
752 rte_eal_mp_wait_lcore();
753 rte_hash_free(tbl_rwc_test_param.h);
759 * Reader(s) lookup keys absent in the table while
760 * 'Main' thread adds with no key-shifts.
763 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
764 int htm, int ext_bkt)
769 uint8_t write_type = WRITE_NO_KEY_SHIFT;
770 uint8_t read_type = READ_FAIL;
773 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
775 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
776 for (m = 0; m < 2; m++) {
778 printf("\n** With bulk-lookup **\n");
779 read_type |= BULK_LOOKUP;
781 for (n = 0; n < NUM_TEST; n++) {
782 unsigned int tot_lcore = rte_lcore_count();
783 if (tot_lcore < rwc_core_cnt[n] + 1)
786 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
788 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
789 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
791 rte_hash_reset(tbl_rwc_test_param.h);
794 for (i = 1; i <= rwc_core_cnt[n]; i++)
795 rte_eal_remote_launch(test_rwc_reader,
796 (void *)(uintptr_t)read_type,
797 enabled_core_ids[i]);
798 ret = write_keys(write_type);
803 for (i = 1; i <= rwc_core_cnt[n]; i++)
804 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
807 unsigned long long cycles_per_lookup =
808 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
809 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
810 rwc_perf_results->w_no_ks_r_miss[m][n]
812 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
817 rte_hash_free(tbl_rwc_test_param.h);
821 rte_eal_mp_wait_lcore();
822 rte_hash_free(tbl_rwc_test_param.h);
828 * Reader(s) lookup keys present in the table and not likely to be on the
829 * shift path while 'Main' thread adds keys causing key-shifts.
832 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
833 int rwc_lf, int htm, int ext_bkt)
840 uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
842 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
844 printf("\nTest: Hash add - key shift, Hash lookup - hit"
845 " (non-shift-path)\n");
846 for (m = 0; m < 2; m++) {
848 printf("\n** With bulk-lookup **\n");
849 read_type |= BULK_LOOKUP;
851 for (n = 0; n < NUM_TEST; n++) {
852 unsigned int tot_lcore = rte_lcore_count();
853 if (tot_lcore < rwc_core_cnt[n] + 1)
856 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
858 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
859 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
861 rte_hash_reset(tbl_rwc_test_param.h);
863 write_type = WRITE_NO_KEY_SHIFT;
864 if (write_keys(write_type) < 0)
866 for (i = 1; i <= rwc_core_cnt[n]; i++)
867 rte_eal_remote_launch(test_rwc_reader,
868 (void *)(uintptr_t)read_type,
869 enabled_core_ids[i]);
870 write_type = WRITE_KEY_SHIFT;
871 ret = write_keys(write_type);
876 for (i = 1; i <= rwc_core_cnt[n]; i++)
877 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
880 unsigned long long cycles_per_lookup =
881 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
882 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
883 rwc_perf_results->w_ks_r_hit_nsp[m][n]
885 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
890 rte_hash_free(tbl_rwc_test_param.h);
894 rte_eal_mp_wait_lcore();
895 rte_hash_free(tbl_rwc_test_param.h);
901 * Reader(s) lookup keys present in the table and likely on the shift-path while
902 * 'Main' thread adds keys causing key-shifts.
905 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
906 int htm, int ext_bkt)
913 uint8_t read_type = READ_PASS_SHIFT_PATH;
915 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
917 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
920 for (m = 0; m < 2; m++) {
922 printf("\n** With bulk-lookup **\n");
923 read_type |= BULK_LOOKUP;
925 for (n = 0; n < NUM_TEST; n++) {
926 unsigned int tot_lcore = rte_lcore_count();
927 if (tot_lcore < rwc_core_cnt[n] + 1)
930 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
932 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
933 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
935 rte_hash_reset(tbl_rwc_test_param.h);
937 write_type = WRITE_NO_KEY_SHIFT;
938 if (write_keys(write_type) < 0)
940 for (i = 1; i <= rwc_core_cnt[n]; i++)
941 rte_eal_remote_launch(test_rwc_reader,
942 (void *)(uintptr_t)read_type,
943 enabled_core_ids[i]);
944 write_type = WRITE_KEY_SHIFT;
945 ret = write_keys(write_type);
950 for (i = 1; i <= rwc_core_cnt[n]; i++)
951 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
954 unsigned long long cycles_per_lookup =
955 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
956 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
957 rwc_perf_results->w_ks_r_hit_sp[m][n]
959 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
964 rte_hash_free(tbl_rwc_test_param.h);
968 rte_eal_mp_wait_lcore();
969 rte_hash_free(tbl_rwc_test_param.h);
975 * Reader(s) lookup keys absent in the table while
976 * 'Main' thread adds keys causing key-shifts.
979 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
987 uint8_t read_type = READ_FAIL;
989 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
991 printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
992 for (m = 0; m < 2; m++) {
994 printf("\n** With bulk-lookup **\n");
995 read_type |= BULK_LOOKUP;
997 for (n = 0; n < NUM_TEST; n++) {
998 unsigned int tot_lcore = rte_lcore_count();
999 if (tot_lcore < rwc_core_cnt[n] + 1)
1002 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1004 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1005 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
1007 rte_hash_reset(tbl_rwc_test_param.h);
1009 write_type = WRITE_NO_KEY_SHIFT;
1010 if (write_keys(write_type) < 0)
1012 for (i = 1; i <= rwc_core_cnt[n]; i++)
1013 rte_eal_remote_launch(test_rwc_reader,
1014 (void *)(uintptr_t)read_type,
1015 enabled_core_ids[i]);
1016 write_type = WRITE_KEY_SHIFT;
1017 ret = write_keys(write_type);
1022 for (i = 1; i <= rwc_core_cnt[n]; i++)
1023 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1026 unsigned long long cycles_per_lookup =
1027 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
1028 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
1029 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1030 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1035 rte_hash_free(tbl_rwc_test_param.h);
1039 rte_eal_mp_wait_lcore();
1040 rte_hash_free(tbl_rwc_test_param.h);
1045 * Test lookup perf for multi-writer:
1046 * Reader(s) lookup keys present in the table and likely on the shift-path while
1047 * Writers add keys causing key-shiftsi.
1048 * Writers are running in parallel, on different data plane cores.
1051 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1052 int htm, int ext_bkt)
1054 unsigned int n, m, k;
1058 uint8_t read_type = READ_PASS_SHIFT_PATH;
1060 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1062 printf("\nTest: Multi-add-lookup\n");
1064 for (m = 1; m < NUM_TEST; m++) {
1065 /* Calculate keys added by each writer */
1066 tbl_rwc_test_param.single_insert =
1067 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1068 for (k = 0; k < 2; k++) {
1070 printf("\n** With bulk-lookup **\n");
1071 read_type |= BULK_LOOKUP;
1073 for (n = 0; n < NUM_TEST; n++) {
1074 unsigned int tot_lcore = rte_lcore_count();
1075 if (tot_lcore < (rwc_core_cnt[n] +
1076 rwc_core_cnt[m] + 1))
1079 printf("\nNumber of writers: %u",
1081 printf("\nNumber of readers: %u\n",
1084 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1085 __atomic_store_n(&gread_cycles, 0,
1088 rte_hash_reset(tbl_rwc_test_param.h);
1090 write_type = WRITE_NO_KEY_SHIFT;
1091 if (write_keys(write_type) < 0)
1094 /* Launch reader(s) */
1095 for (i = 1; i <= rwc_core_cnt[n]; i++)
1096 rte_eal_remote_launch(test_rwc_reader,
1097 (void *)(uintptr_t)read_type,
1098 enabled_core_ids[i]);
1099 write_type = WRITE_KEY_SHIFT;
1102 /* Launch writers */
1103 for (; i <= rwc_core_cnt[m]
1104 + rwc_core_cnt[n]; i++) {
1105 rte_eal_remote_launch
1106 (test_rwc_multi_writer,
1107 (void *)(uintptr_t)pos_core,
1108 enabled_core_ids[i]);
1112 /* Wait for writers to complete */
1113 for (i = rwc_core_cnt[n] + 1;
1114 i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1116 rte_eal_wait_lcore(enabled_core_ids[i]);
1120 for (i = 1; i <= rwc_core_cnt[n]; i++)
1121 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1124 unsigned long long cycles_per_lookup =
1125 __atomic_load_n(&gread_cycles,
1127 __atomic_load_n(&greads,
1129 rwc_perf_results->multi_rw[m][k][n]
1130 = cycles_per_lookup;
1131 printf("Cycles per lookup: %llu\n",
1138 rte_hash_free(tbl_rwc_test_param.h);
1142 rte_eal_mp_wait_lcore();
1143 rte_hash_free(tbl_rwc_test_param.h);
1149 * Reader(s) lookup keys present in the extendable bkt.
1152 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1153 int rwc_lf, int htm, int ext_bkt)
1159 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1161 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1163 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1164 for (m = 0; m < 2; m++) {
1166 printf("\n** With bulk-lookup **\n");
1167 read_type |= BULK_LOOKUP;
1169 for (n = 0; n < NUM_TEST; n++) {
1170 unsigned int tot_lcore = rte_lcore_count();
1171 if (tot_lcore < rwc_core_cnt[n] + 1)
1174 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1176 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1177 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
1179 rte_hash_reset(tbl_rwc_test_param.h);
1180 write_type = WRITE_NO_KEY_SHIFT;
1181 if (write_keys(write_type) < 0)
1183 write_type = WRITE_KEY_SHIFT;
1184 if (write_keys(write_type) < 0)
1187 for (i = 1; i <= rwc_core_cnt[n]; i++)
1188 rte_eal_remote_launch(test_rwc_reader,
1189 (void *)(uintptr_t)read_type,
1190 enabled_core_ids[i]);
1191 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1193 if (rte_hash_del_key(tbl_rwc_test_param.h,
1194 tbl_rwc_test_param.keys_ks_extbkt + i)
1196 printf("Delete Failed: %u\n",
1197 tbl_rwc_test_param.keys_ks_extbkt[i]);
1203 for (i = 1; i <= rwc_core_cnt[n]; i++)
1204 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1207 unsigned long long cycles_per_lookup =
1208 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
1209 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
1210 rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1211 = cycles_per_lookup;
1212 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1217 rte_hash_free(tbl_rwc_test_param.h);
1221 rte_eal_mp_wait_lcore();
1222 rte_hash_free(tbl_rwc_test_param.h);
1227 test_hash_readwrite_lf_perf_main(void)
1230 * Variables used to choose different tests.
1231 * rwc_lf indicates if read-write concurrency lock-free support is
1233 * htm indicates if Hardware transactional memory support is enabled.
1239 if (rte_lcore_count() < 2) {
1240 printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n");
1241 return TEST_SKIPPED;
1244 setlocale(LC_NUMERIC, "");
1246 /* Reset tbl_rwc_test_param to discard values from previous run */
1247 memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param));
1249 if (rte_tm_supported())
1254 if (generate_keys() != 0)
1256 if (get_enabled_cores_list() != 0)
1259 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1262 printf("Test lookup with read-write concurrency lock free support"
1264 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1267 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1270 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1273 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1276 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1279 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1282 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1286 printf("\nTest lookup with read-write concurrency lock free support"
1290 printf("With HTM Disabled\n");
1291 if (!RUN_WITH_HTM_DISABLED) {
1292 printf("Enable RUN_WITH_HTM_DISABLED to test with"
1293 " lock-free disabled");
1297 printf("With HTM Enabled\n");
1298 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1301 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1304 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1307 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1310 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1313 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1316 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1320 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1322 for (j = 0; j < 2; j++) {
1324 printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1325 "**********#######\n\n");
1326 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1327 "\t\t\t\t_________________\n");
1328 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1329 "\t\t\tCycles per lookup\n");
1330 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1331 "\t\t\t_________________\n");
1332 for (i = 0; i < NUM_TEST; i++) {
1333 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1334 printf("Enabled\t\t");
1336 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1337 "%u\n\t\t\t\t\t\t\t\t",
1338 rwc_lf_results.w_no_ks_r_hit[j][i]);
1339 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1340 "%u\n\t\t\t\t\t\t\t\t",
1341 rwc_lf_results.w_no_ks_r_miss[j][i]);
1342 printf("Hash add - key-shifts, lookup - hit"
1343 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1344 rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1345 printf("Hash add - key-shifts, lookup - hit "
1346 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1347 rwc_lf_results.w_ks_r_hit_sp[j][i]);
1348 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1349 "%u\n\t\t\t\t\t\t\t\t",
1350 rwc_lf_results.w_ks_r_miss[j][i]);
1351 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1353 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1355 printf("Disabled\t");
1357 printf("Enabled\t\t");
1359 printf("Disabled\t");
1360 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1361 "%u\n\t\t\t\t\t\t\t\t",
1362 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1363 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1364 "%u\n\t\t\t\t\t\t\t\t",
1365 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1366 printf("Hash add - key-shifts, lookup - hit "
1367 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1368 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1369 printf("Hash add - key-shifts, lookup - hit "
1370 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1371 rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1372 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1373 "%u\n\t\t\t\t\t\t\t\t",
1374 rwc_non_lf_results.w_ks_r_miss[j][i]);
1375 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1377 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1379 printf("_______\t\t_______\t\t_________\t___\t\t"
1380 "_________\t\t\t\t\t\t_________________\n");
1383 for (i = 1; i < NUM_TEST; i++) {
1384 for (k = 0; k < NUM_TEST; k++) {
1385 printf("%u", rwc_core_cnt[i]);
1386 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1387 printf("Enabled\t\t");
1389 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1391 rwc_lf_results.multi_rw[i][j][k]);
1392 printf("Disabled\t");
1394 printf("Enabled\t\t");
1396 printf("Disabled\t");
1397 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1398 rwc_non_lf_results.multi_rw[i][j][k]);
1400 printf("_______\t\t_______\t\t_________\t___"
1401 "\t\t_________\t\t\t\t\t\t"
1402 "_________________\n");
1406 rte_free(tbl_rwc_test_param.keys);
1407 rte_free(tbl_rwc_test_param.keys_no_ks);
1408 rte_free(tbl_rwc_test_param.keys_ks);
1409 rte_free(tbl_rwc_test_param.keys_absent);
1410 rte_free(tbl_rwc_test_param.keys_shift_path);
1411 rte_free(tbl_rwc_test_param.keys_non_shift_path);
1412 rte_free(tbl_rwc_test_param.keys_ext_bkt);
1413 rte_free(tbl_rwc_test_param.keys_ks_extbkt);
1417 REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest,
1418 test_hash_readwrite_lf_perf_main);