1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Arm Limited
8 #include <rte_cycles.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
23 #define BULK_LOOKUP_SIZE 32
25 #define RUN_WITH_HTM_DISABLED 0
27 #if (RUN_WITH_HTM_DISABLED)
29 #define TOTAL_ENTRY (5*1024)
30 #define TOTAL_INSERT (5*1024)
34 #define TOTAL_ENTRY (4*1024*1024)
35 #define TOTAL_INSERT (4*1024*1024)
40 #define READ_PASS_NO_KEY_SHIFTS 2
41 #define READ_PASS_SHIFT_PATH 4
42 #define READ_PASS_NON_SHIFT_PATH 8
43 #define BULK_LOOKUP 16
44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
46 #define WRITE_NO_KEY_SHIFT 0
47 #define WRITE_KEY_SHIFT 1
48 #define WRITE_EXT_BKT 2
51 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
54 uint32_t w_no_ks_r_hit[2][NUM_TEST];
55 uint32_t w_no_ks_r_miss[2][NUM_TEST];
56 uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
57 uint32_t w_ks_r_hit_sp[2][NUM_TEST];
58 uint32_t w_ks_r_miss[2][NUM_TEST];
59 uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
60 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
69 uint32_t *keys_absent;
70 uint32_t *keys_shift_path;
71 uint32_t *keys_non_shift_path;
72 uint32_t *keys_ext_bkt;
73 uint32_t *keys_ks_extbkt;
74 uint32_t count_keys_no_ks;
75 uint32_t count_keys_ks;
76 uint32_t count_keys_absent;
77 uint32_t count_keys_shift_path;
78 uint32_t count_keys_non_shift_path;
79 uint32_t count_keys_extbkt;
80 uint32_t count_keys_ks_extbkt;
81 uint32_t single_insert;
85 static rte_atomic64_t gread_cycles;
86 static rte_atomic64_t greads;
88 static volatile uint8_t writer_done;
90 uint16_t enabled_core_ids[RTE_MAX_LCORE];
92 uint8_t *scanned_bkts;
94 static inline uint16_t
95 get_short_sig(const hash_sig_t hash)
100 static inline uint32_t
101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
102 const hash_sig_t hash)
104 uint32_t num_buckets;
105 uint32_t bucket_bitmask;
106 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
107 bucket_bitmask = num_buckets - 1;
108 return hash & bucket_bitmask;
111 static inline uint32_t
112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
113 uint32_t cur_bkt_idx, uint16_t sig)
115 uint32_t num_buckets;
116 uint32_t bucket_bitmask;
117 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
118 bucket_bitmask = num_buckets - 1;
119 return (cur_bkt_idx ^ sig) & bucket_bitmask;
124 get_enabled_cores_list(void)
128 uint32_t max_cores = rte_lcore_count();
129 RTE_LCORE_FOREACH(core_id) {
130 enabled_core_ids[i] = core_id;
134 if (i != max_cores) {
135 printf("Number of enabled cores in list is different from "
136 "number given by rte_lcore_count()\n");
143 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
145 struct rte_hash *handle;
147 struct rte_hash_parameters hash_params = {
148 .entries = TOTAL_ENTRY,
149 .key_len = sizeof(uint32_t),
150 .hash_func_init_val = 0,
151 .socket_id = rte_socket_id(),
155 hash_params.hash_func = rte_jhash;
157 hash_params.hash_func = rte_hash_crc;
160 hash_params.extra_flag =
161 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
162 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
164 hash_params.extra_flag =
165 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
166 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
167 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
169 hash_params.extra_flag =
170 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
171 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
174 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
176 hash_params.name = "tests";
178 handle = rte_hash_create(&hash_params);
179 if (handle == NULL) {
180 printf("hash creation failed");
184 tbl_rwc_test_param.h = handle;
189 check_bucket(uint32_t bkt_idx, uint32_t key)
195 const void *next_key;
198 /* Temporary bucket to hold the keys */
199 uint32_t keys_in_bkt[8];
203 while (rte_hash_iterate(tbl_rwc_test_param.h,
204 &next_key, &next_data, &iter) >= 0) {
206 /* Check for duplicate entries */
207 if (*(const uint32_t *)next_key == key)
210 /* Identify if there is any free entry in the bucket */
211 diff = iter - prev_iter;
216 keys_in_bkt[count] = *(const uint32_t *)next_key;
219 /* All entries in the bucket are occupied */
223 * Check if bucket was not scanned before, to avoid
226 if (scanned_bkts[bkt_idx] == 0) {
228 * Since this bucket (pointed to by bkt_idx) is
229 * full, it is likely that key(s) in this
230 * bucket will be on the shift path, when
231 * collision occurs. Thus, add it to
234 memcpy(tbl_rwc_test_param.keys_shift_path +
235 tbl_rwc_test_param.count_keys_shift_path
237 tbl_rwc_test_param.count_keys_shift_path += 8;
238 scanned_bkts[bkt_idx] = 1;
249 uint32_t *keys = NULL;
250 uint32_t *keys_no_ks = NULL;
251 uint32_t *keys_ks = NULL;
252 uint32_t *keys_absent = NULL;
253 uint32_t *keys_non_shift_path = NULL;
254 uint32_t *keys_ext_bkt = NULL;
255 uint32_t *keys_ks_extbkt = NULL;
256 uint32_t *found = NULL;
257 uint32_t count_keys_no_ks = 0;
258 uint32_t count_keys_ks = 0;
259 uint32_t count_keys_extbkt = 0;
262 if (init_params(0, 0, 0, 0) != 0)
266 * keys will consist of a) keys whose addition to the hash table
267 * will result in shifting of the existing keys to their alternate
268 * locations b) keys whose addition to the hash table will not result
269 * in shifting of the existing keys.
271 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
273 printf("RTE_MALLOC failed\n");
278 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that
279 * will NOT result in shifting of the existing keys to their alternate
280 * locations. Roughly around 900K keys.
282 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
283 if (keys_no_ks == NULL) {
284 printf("RTE_MALLOC failed\n");
289 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
290 * result in shifting of the existing keys to their alternate locations.
291 * Roughly around 146K keys. There might be repeating keys. More code is
292 * required to filter out these keys which will complicate the test case
294 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
295 if (keys_ks == NULL) {
296 printf("RTE_MALLOC failed\n");
300 /* Used to identify keys not inserted in the hash table */
301 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
303 printf("RTE_MALLOC failed\n");
308 * This consist of keys not inserted to the hash table.
309 * Used to test perf of lookup on keys that do not exist in the table.
311 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
312 if (keys_absent == NULL) {
313 printf("RTE_MALLOC failed\n");
318 * This consist of keys which are likely to be on the shift
319 * path (i.e. being moved to alternate location), when collision occurs
320 * on addition of a key to an already full primary bucket.
321 * Used to test perf of lookup on keys that are on the shift path.
323 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
325 if (tbl_rwc_test_param.keys_shift_path == NULL) {
326 printf("RTE_MALLOC failed\n");
331 * This consist of keys which are never on the shift
332 * path (i.e. being moved to alternate location), when collision occurs
333 * on addition of a key to an already full primary bucket.
334 * Used to test perf of lookup on keys that are not on the shift path.
336 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
338 if (keys_non_shift_path == NULL) {
339 printf("RTE_MALLOC failed\n");
344 * This consist of keys which will be stored in extended buckets
346 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
347 if (keys_ext_bkt == NULL) {
348 printf("RTE_MALLOC failed\n");
353 * This consist of keys which when deleted causes shifting of keys
354 * in extended buckets to respective secondary buckets
356 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
357 if (keys_ks_extbkt == NULL) {
358 printf("RTE_MALLOC failed\n");
363 uint32_t prim_bucket_idx;
364 uint32_t sec_bucket_idx;
366 uint32_t num_buckets;
367 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
371 * Used to mark bkts in which at least one key was shifted to its
374 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
375 if (scanned_bkts == NULL) {
376 printf("RTE_MALLOC failed\n");
380 tbl_rwc_test_param.keys = keys;
381 tbl_rwc_test_param.keys_no_ks = keys_no_ks;
382 tbl_rwc_test_param.keys_ks = keys_ks;
383 tbl_rwc_test_param.keys_absent = keys_absent;
384 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
385 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
386 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
387 /* Generate keys by adding previous two keys, neglect overflow */
388 printf("Generating keys...\n");
391 for (i = 2; i < TOTAL_INSERT; i++)
392 keys[i] = keys[i-1] + keys[i-2];
394 /* Segregate keys into keys_no_ks and keys_ks */
395 for (i = 0; i < TOTAL_INSERT; i++) {
396 /* Check if primary bucket has space.*/
397 sig = rte_hash_hash(tbl_rwc_test_param.h,
398 tbl_rwc_test_param.keys+i);
399 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
401 ret = check_bucket(prim_bucket_idx, keys[i]);
404 * Primary bucket is full, this key will result in
405 * shifting of the keys to their alternate locations.
407 keys_ks[count_keys_ks] = keys[i];
409 } else if (ret == 0) {
411 * Primary bucket has space, this key will not result in
412 * shifting of the keys. Hence, add key to the table.
414 ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
416 (void *)((uintptr_t)i));
418 printf("writer failed %"PRIu32"\n", i);
421 keys_no_ks[count_keys_no_ks] = keys[i];
426 for (i = 0; i < count_keys_no_ks; i++) {
428 * Identify keys in keys_no_ks with value less than
429 * 4M (HTM enabled) OR 5K (HTM disabled)
431 if (keys_no_ks[i] < TOTAL_INSERT)
432 found[keys_no_ks[i]]++;
435 for (i = 0; i < count_keys_ks; i++) {
437 * Identify keys in keys_ks with value less than
438 * 4M (HTM enabled) OR 5K (HTM disabled)
440 if (keys_ks[i] < TOTAL_INSERT)
444 uint32_t count_keys_absent = 0;
445 for (i = 0; i < TOTAL_INSERT; i++) {
447 * Identify missing keys between 0 and
448 * 4M (HTM enabled) OR 5K (HTM disabled)
451 keys_absent[count_keys_absent++] = i;
454 /* Find keys that will not be on the shift path */
456 const void *next_key;
459 for (i = 0; i < num_buckets; i++) {
460 /* Check bucket for no keys shifted to alternate locations */
461 if (scanned_bkts[i] == 0) {
463 while (rte_hash_iterate(tbl_rwc_test_param.h,
464 &next_key, &next_data, &iter) >= 0) {
466 /* Check if key belongs to the current bucket */
468 keys_non_shift_path[count++]
469 = *(const uint32_t *)next_key;
476 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
477 tbl_rwc_test_param.count_keys_ks = count_keys_ks;
478 tbl_rwc_test_param.count_keys_absent = count_keys_absent;
479 tbl_rwc_test_param.count_keys_non_shift_path = count;
481 memset(scanned_bkts, 0, num_buckets);
483 /* Find keys that will be in extended buckets */
484 for (i = 0; i < count_keys_ks; i++) {
485 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
487 /* Key will be added to ext bkt */
488 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
489 /* Sec bkt to be added to keys_ks_extbkt */
490 sig = rte_hash_hash(tbl_rwc_test_param.h,
491 tbl_rwc_test_param.keys_ks + i);
492 prim_bucket_idx = get_prim_bucket_index(
493 tbl_rwc_test_param.h, sig);
494 short_sig = get_short_sig(sig);
495 sec_bucket_idx = get_alt_bucket_index(
496 tbl_rwc_test_param.h,
497 prim_bucket_idx, short_sig);
498 if (scanned_bkts[sec_bucket_idx] == 0)
499 scanned_bkts[sec_bucket_idx] = 1;
503 /* Find keys that will shift keys in ext bucket*/
504 for (i = 0; i < num_buckets; i++) {
505 if (scanned_bkts[i] == 1) {
507 while (rte_hash_iterate(tbl_rwc_test_param.h,
508 &next_key, &next_data, &iter) >= 0) {
509 /* Check if key belongs to the current bucket */
511 keys_ks_extbkt[count++]
512 = *(const uint32_t *)next_key;
519 tbl_rwc_test_param.count_keys_ks_extbkt = count;
520 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
522 printf("\nCount of keys NOT causing shifting of existing keys to "
523 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
524 printf("\nCount of keys causing shifting of existing keys to alternate "
525 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
526 printf("Count of absent keys that will never be added to the hash "
527 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
528 printf("Count of keys likely to be on the shift path: %d\n\n",
529 tbl_rwc_test_param.count_keys_shift_path);
530 printf("Count of keys not likely to be on the shift path: %d\n\n",
531 tbl_rwc_test_param.count_keys_non_shift_path);
532 printf("Count of keys in extended buckets: %d\n\n",
533 tbl_rwc_test_param.count_keys_extbkt);
534 printf("Count of keys shifting keys in ext buckets: %d\n\n",
535 tbl_rwc_test_param.count_keys_ks_extbkt);
538 rte_free(scanned_bkts);
539 rte_hash_free(tbl_rwc_test_param.h);
544 rte_free(keys_no_ks);
546 rte_free(keys_absent);
548 rte_free(tbl_rwc_test_param.keys_shift_path);
549 rte_free(keys_non_shift_path);
550 rte_free(keys_ext_bkt);
551 rte_free(keys_ks_extbkt);
552 rte_free(scanned_bkts);
553 rte_hash_free(tbl_rwc_test_param.h);
558 test_rwc_reader(__attribute__((unused)) void *arg)
562 uint64_t begin, cycles;
563 uint32_t loop_cnt = 0;
564 uint8_t read_type = (uint8_t)((uintptr_t)arg);
568 int32_t pos[BULK_LOOKUP_SIZE];
569 void *temp_a[BULK_LOOKUP_SIZE];
571 if (read_type & READ_FAIL) {
572 keys = tbl_rwc_test_param.keys_absent;
573 read_cnt = tbl_rwc_test_param.count_keys_absent;
574 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
575 keys = tbl_rwc_test_param.keys_no_ks;
576 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
577 } else if (read_type & READ_PASS_SHIFT_PATH) {
578 keys = tbl_rwc_test_param.keys_shift_path;
579 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
580 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
581 keys = tbl_rwc_test_param.keys_ext_bkt;
582 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
584 keys = tbl_rwc_test_param.keys_non_shift_path;
585 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
588 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
590 begin = rte_rdtsc_precise();
592 if (read_type & BULK_LOOKUP) {
593 for (i = 0; i < (read_cnt - extra_keys);
594 i += BULK_LOOKUP_SIZE) {
595 /* Array of pointer to the list of keys */
596 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
597 temp_a[j] = keys + i + j;
598 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
601 BULK_LOOKUP_SIZE, pos);
602 /* Validate lookup result */
603 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
604 if ((read_type & READ_FAIL &&
605 pos[j] != -ENOENT) ||
606 (!(read_type & READ_FAIL) &&
607 pos[j] == -ENOENT)) {
608 printf("lookup failed!"
614 for (j = 0; j < extra_keys; j++)
615 temp_a[j] = keys + i + j;
617 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
621 for (j = 0; j < extra_keys; j++)
622 if ((read_type & READ_FAIL &&
623 pos[j] != -ENOENT) ||
624 (!(read_type & READ_FAIL) &&
625 pos[j] == -ENOENT)) {
626 printf("lookup failed! %"PRIu32"\n",
631 for (i = 0; i < read_cnt; i++) {
632 ret = rte_hash_lookup
633 (tbl_rwc_test_param.h, keys + i);
634 if (((read_type & READ_FAIL) &&
636 (!(read_type & READ_FAIL) &&
638 printf("lookup failed! %"PRIu32"\n",
645 } while (!writer_done);
647 cycles = rte_rdtsc_precise() - begin;
648 rte_atomic64_add(&gread_cycles, cycles);
649 rte_atomic64_add(&greads, read_cnt*loop_cnt);
654 write_keys(uint8_t write_type)
658 uint32_t key_cnt = 0;
660 if (write_type == WRITE_KEY_SHIFT) {
661 key_cnt = tbl_rwc_test_param.count_keys_ks;
662 keys = tbl_rwc_test_param.keys_ks;
663 } else if (write_type == WRITE_NO_KEY_SHIFT) {
664 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
665 keys = tbl_rwc_test_param.keys_no_ks;
666 } else if (write_type == WRITE_EXT_BKT) {
667 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
668 keys = tbl_rwc_test_param.keys_ext_bkt;
670 for (i = 0; i < key_cnt; i++) {
671 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
672 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
673 printf("writer failed %"PRIu32"\n", i);
681 test_rwc_multi_writer(__attribute__((unused)) void *arg)
684 uint32_t pos_core = (uint32_t)((uintptr_t)arg);
685 offset = pos_core * tbl_rwc_test_param.single_insert;
686 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
687 rte_hash_add_key(tbl_rwc_test_param.h,
688 tbl_rwc_test_param.keys_ks + i);
694 * Reader(s) lookup keys present in the table.
697 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
698 int htm, int ext_bkt)
703 uint8_t write_type = WRITE_NO_KEY_SHIFT;
704 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
706 rte_atomic64_init(&greads);
707 rte_atomic64_init(&gread_cycles);
709 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
711 printf("\nTest: Hash add - no key-shifts, read - hit\n");
712 for (m = 0; m < 2; m++) {
714 printf("\n** With bulk-lookup **\n");
715 read_type |= BULK_LOOKUP;
717 for (n = 0; n < NUM_TEST; n++) {
718 unsigned int tot_lcore = rte_lcore_count();
719 if (tot_lcore < rwc_core_cnt[n] + 1)
722 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
724 rte_atomic64_clear(&greads);
725 rte_atomic64_clear(&gread_cycles);
727 rte_hash_reset(tbl_rwc_test_param.h);
729 if (write_keys(write_type) < 0)
732 for (i = 1; i <= rwc_core_cnt[n]; i++)
733 rte_eal_remote_launch(test_rwc_reader,
734 (void *)(uintptr_t)read_type,
735 enabled_core_ids[i]);
737 for (i = 1; i <= rwc_core_cnt[n]; i++)
738 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
741 unsigned long long cycles_per_lookup =
742 rte_atomic64_read(&gread_cycles) /
743 rte_atomic64_read(&greads);
744 rwc_perf_results->w_no_ks_r_hit[m][n]
746 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
751 rte_hash_free(tbl_rwc_test_param.h);
755 rte_eal_mp_wait_lcore();
756 rte_hash_free(tbl_rwc_test_param.h);
762 * Reader(s) lookup keys absent in the table while
763 * 'Main' thread adds with no key-shifts.
766 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
767 int htm, int ext_bkt)
772 uint8_t write_type = WRITE_NO_KEY_SHIFT;
773 uint8_t read_type = READ_FAIL;
776 rte_atomic64_init(&greads);
777 rte_atomic64_init(&gread_cycles);
779 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
781 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
782 for (m = 0; m < 2; m++) {
784 printf("\n** With bulk-lookup **\n");
785 read_type |= BULK_LOOKUP;
787 for (n = 0; n < NUM_TEST; n++) {
788 unsigned int tot_lcore = rte_lcore_count();
789 if (tot_lcore < rwc_core_cnt[n] + 1)
792 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
794 rte_atomic64_clear(&greads);
795 rte_atomic64_clear(&gread_cycles);
797 rte_hash_reset(tbl_rwc_test_param.h);
800 for (i = 1; i <= rwc_core_cnt[n]; i++)
801 rte_eal_remote_launch(test_rwc_reader,
802 (void *)(uintptr_t)read_type,
803 enabled_core_ids[i]);
804 ret = write_keys(write_type);
809 for (i = 1; i <= rwc_core_cnt[n]; i++)
810 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
813 unsigned long long cycles_per_lookup =
814 rte_atomic64_read(&gread_cycles) /
815 rte_atomic64_read(&greads);
816 rwc_perf_results->w_no_ks_r_miss[m][n]
818 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
823 rte_hash_free(tbl_rwc_test_param.h);
827 rte_eal_mp_wait_lcore();
828 rte_hash_free(tbl_rwc_test_param.h);
834 * Reader(s) lookup keys present in the table and not likely to be on the
835 * shift path while 'Main' thread adds keys causing key-shifts.
838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
839 int rwc_lf, int htm, int ext_bkt)
846 uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
848 rte_atomic64_init(&greads);
849 rte_atomic64_init(&gread_cycles);
851 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
853 printf("\nTest: Hash add - key shift, Hash lookup - hit"
854 " (non-shift-path)\n");
855 for (m = 0; m < 2; m++) {
857 printf("\n** With bulk-lookup **\n");
858 read_type |= BULK_LOOKUP;
860 for (n = 0; n < NUM_TEST; n++) {
861 unsigned int tot_lcore = rte_lcore_count();
862 if (tot_lcore < rwc_core_cnt[n] + 1)
865 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
867 rte_atomic64_clear(&greads);
868 rte_atomic64_clear(&gread_cycles);
870 rte_hash_reset(tbl_rwc_test_param.h);
872 write_type = WRITE_NO_KEY_SHIFT;
873 if (write_keys(write_type) < 0)
875 for (i = 1; i <= rwc_core_cnt[n]; i++)
876 rte_eal_remote_launch(test_rwc_reader,
877 (void *)(uintptr_t)read_type,
878 enabled_core_ids[i]);
879 write_type = WRITE_KEY_SHIFT;
880 ret = write_keys(write_type);
885 for (i = 1; i <= rwc_core_cnt[n]; i++)
886 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
889 unsigned long long cycles_per_lookup =
890 rte_atomic64_read(&gread_cycles) /
891 rte_atomic64_read(&greads);
892 rwc_perf_results->w_ks_r_hit_nsp[m][n]
894 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
899 rte_hash_free(tbl_rwc_test_param.h);
903 rte_eal_mp_wait_lcore();
904 rte_hash_free(tbl_rwc_test_param.h);
910 * Reader(s) lookup keys present in the table and likely on the shift-path while
911 * 'Main' thread adds keys causing key-shifts.
914 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
915 int htm, int ext_bkt)
922 uint8_t read_type = READ_PASS_SHIFT_PATH;
924 rte_atomic64_init(&greads);
925 rte_atomic64_init(&gread_cycles);
927 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
929 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
932 for (m = 0; m < 2; m++) {
934 printf("\n** With bulk-lookup **\n");
935 read_type |= BULK_LOOKUP;
937 for (n = 0; n < NUM_TEST; n++) {
938 unsigned int tot_lcore = rte_lcore_count();
939 if (tot_lcore < rwc_core_cnt[n] + 1)
942 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
943 rte_atomic64_clear(&greads);
944 rte_atomic64_clear(&gread_cycles);
946 rte_hash_reset(tbl_rwc_test_param.h);
948 write_type = WRITE_NO_KEY_SHIFT;
949 if (write_keys(write_type) < 0)
951 for (i = 1; i <= rwc_core_cnt[n]; i++)
952 rte_eal_remote_launch(test_rwc_reader,
953 (void *)(uintptr_t)read_type,
954 enabled_core_ids[i]);
955 write_type = WRITE_KEY_SHIFT;
956 ret = write_keys(write_type);
961 for (i = 1; i <= rwc_core_cnt[n]; i++)
962 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
965 unsigned long long cycles_per_lookup =
966 rte_atomic64_read(&gread_cycles) /
967 rte_atomic64_read(&greads);
968 rwc_perf_results->w_ks_r_hit_sp[m][n]
970 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
975 rte_hash_free(tbl_rwc_test_param.h);
979 rte_eal_mp_wait_lcore();
980 rte_hash_free(tbl_rwc_test_param.h);
986 * Reader(s) lookup keys absent in the table while
987 * 'Main' thread adds keys causing key-shifts.
990 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
998 uint8_t read_type = READ_FAIL;
1000 rte_atomic64_init(&greads);
1001 rte_atomic64_init(&gread_cycles);
1003 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1005 printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1006 for (m = 0; m < 2; m++) {
1008 printf("\n** With bulk-lookup **\n");
1009 read_type |= BULK_LOOKUP;
1011 for (n = 0; n < NUM_TEST; n++) {
1012 unsigned int tot_lcore = rte_lcore_count();
1013 if (tot_lcore < rwc_core_cnt[n] + 1)
1016 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1018 rte_atomic64_clear(&greads);
1019 rte_atomic64_clear(&gread_cycles);
1021 rte_hash_reset(tbl_rwc_test_param.h);
1023 write_type = WRITE_NO_KEY_SHIFT;
1024 if (write_keys(write_type) < 0)
1026 for (i = 1; i <= rwc_core_cnt[n]; i++)
1027 rte_eal_remote_launch(test_rwc_reader,
1028 (void *)(uintptr_t)read_type,
1029 enabled_core_ids[i]);
1030 write_type = WRITE_KEY_SHIFT;
1031 ret = write_keys(write_type);
1036 for (i = 1; i <= rwc_core_cnt[n]; i++)
1037 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1040 unsigned long long cycles_per_lookup =
1041 rte_atomic64_read(&gread_cycles) /
1042 rte_atomic64_read(&greads);
1043 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1044 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1049 rte_hash_free(tbl_rwc_test_param.h);
1053 rte_eal_mp_wait_lcore();
1054 rte_hash_free(tbl_rwc_test_param.h);
1059 * Test lookup perf for multi-writer:
1060 * Reader(s) lookup keys present in the table and likely on the shift-path while
1061 * Writers add keys causing key-shiftsi.
1062 * Writers are running in parallel, on different data plane cores.
1065 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1066 int htm, int ext_bkt)
1068 unsigned int n, m, k;
1072 uint8_t read_type = READ_PASS_SHIFT_PATH;
1074 rte_atomic64_init(&greads);
1075 rte_atomic64_init(&gread_cycles);
1077 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1079 printf("\nTest: Multi-add-lookup\n");
1081 for (m = 1; m < NUM_TEST; m++) {
1082 /* Calculate keys added by each writer */
1083 tbl_rwc_test_param.single_insert =
1084 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1085 for (k = 0; k < 2; k++) {
1087 printf("\n** With bulk-lookup **\n");
1088 read_type |= BULK_LOOKUP;
1090 for (n = 0; n < NUM_TEST; n++) {
1091 unsigned int tot_lcore = rte_lcore_count();
1092 if (tot_lcore < (rwc_core_cnt[n] +
1093 rwc_core_cnt[m] + 1))
1096 printf("\nNumber of writers: %u",
1098 printf("\nNumber of readers: %u\n",
1101 rte_atomic64_clear(&greads);
1102 rte_atomic64_clear(&gread_cycles);
1104 rte_hash_reset(tbl_rwc_test_param.h);
1106 write_type = WRITE_NO_KEY_SHIFT;
1107 if (write_keys(write_type) < 0)
1110 /* Launch reader(s) */
1111 for (i = 1; i <= rwc_core_cnt[n]; i++)
1112 rte_eal_remote_launch(test_rwc_reader,
1113 (void *)(uintptr_t)read_type,
1114 enabled_core_ids[i]);
1115 write_type = WRITE_KEY_SHIFT;
1118 /* Launch writers */
1119 for (; i <= rwc_core_cnt[m]
1120 + rwc_core_cnt[n]; i++) {
1121 rte_eal_remote_launch
1122 (test_rwc_multi_writer,
1123 (void *)(uintptr_t)pos_core,
1124 enabled_core_ids[i]);
1128 /* Wait for writers to complete */
1129 for (i = rwc_core_cnt[n] + 1;
1130 i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1132 rte_eal_wait_lcore(enabled_core_ids[i]);
1136 for (i = 1; i <= rwc_core_cnt[n]; i++)
1137 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1140 unsigned long long cycles_per_lookup =
1141 rte_atomic64_read(&gread_cycles)
1142 / rte_atomic64_read(&greads);
1143 rwc_perf_results->multi_rw[m][k][n]
1144 = cycles_per_lookup;
1145 printf("Cycles per lookup: %llu\n",
1152 rte_hash_free(tbl_rwc_test_param.h);
1156 rte_eal_mp_wait_lcore();
1157 rte_hash_free(tbl_rwc_test_param.h);
1163 * Reader(s) lookup keys present in the extendable bkt.
1166 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1167 int rwc_lf, int htm, int ext_bkt)
1173 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1175 rte_atomic64_init(&greads);
1176 rte_atomic64_init(&gread_cycles);
1178 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1180 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1181 for (m = 0; m < 2; m++) {
1183 printf("\n** With bulk-lookup **\n");
1184 read_type |= BULK_LOOKUP;
1186 for (n = 0; n < NUM_TEST; n++) {
1187 unsigned int tot_lcore = rte_lcore_count();
1188 if (tot_lcore < rwc_core_cnt[n] + 1)
1191 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1193 rte_atomic64_clear(&greads);
1194 rte_atomic64_clear(&gread_cycles);
1196 rte_hash_reset(tbl_rwc_test_param.h);
1197 write_type = WRITE_NO_KEY_SHIFT;
1198 if (write_keys(write_type) < 0)
1200 write_type = WRITE_KEY_SHIFT;
1201 if (write_keys(write_type) < 0)
1204 for (i = 1; i <= rwc_core_cnt[n]; i++)
1205 rte_eal_remote_launch(test_rwc_reader,
1206 (void *)(uintptr_t)read_type,
1207 enabled_core_ids[i]);
1208 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1210 if (rte_hash_del_key(tbl_rwc_test_param.h,
1211 tbl_rwc_test_param.keys_ks_extbkt + i)
1213 printf("Delete Failed: %u\n",
1214 tbl_rwc_test_param.keys_ks_extbkt[i]);
1220 for (i = 1; i <= rwc_core_cnt[n]; i++)
1221 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1224 unsigned long long cycles_per_lookup =
1225 rte_atomic64_read(&gread_cycles) /
1226 rte_atomic64_read(&greads);
1227 rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1228 = cycles_per_lookup;
1229 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1234 rte_hash_free(tbl_rwc_test_param.h);
1238 rte_eal_mp_wait_lcore();
1239 rte_hash_free(tbl_rwc_test_param.h);
1244 test_hash_readwrite_lf_main(void)
1247 * Variables used to choose different tests.
1248 * rwc_lf indicates if read-write concurrency lock-free support is
1250 * htm indicates if Hardware transactional memory support is enabled.
1256 if (rte_lcore_count() < 2) {
1257 printf("Not enough cores for hash_readwrite_lf_autotest, expecting at least 2\n");
1258 return TEST_SKIPPED;
1261 setlocale(LC_NUMERIC, "");
1263 /* Reset tbl_rwc_test_param to discard values from previous run */
1264 memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param));
1266 if (rte_tm_supported())
1271 if (generate_keys() != 0)
1273 if (get_enabled_cores_list() != 0)
1276 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1279 printf("Test lookup with read-write concurrency lock free support"
1281 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1284 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1287 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1290 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1293 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1296 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1299 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1303 printf("\nTest lookup with read-write concurrency lock free support"
1307 printf("With HTM Disabled\n");
1308 if (!RUN_WITH_HTM_DISABLED) {
1309 printf("Enable RUN_WITH_HTM_DISABLED to test with"
1310 " lock-free disabled");
1314 printf("With HTM Enabled\n");
1315 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1318 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1321 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1324 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1327 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1330 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1333 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1337 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1339 for (j = 0; j < 2; j++) {
1341 printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1342 "**********#######\n\n");
1343 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1344 "\t\t\t\t_________________\n");
1345 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1346 "\t\t\tCycles per lookup\n");
1347 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1348 "\t\t\t_________________\n");
1349 for (i = 0; i < NUM_TEST; i++) {
1350 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1351 printf("Enabled\t\t");
1353 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1354 "%u\n\t\t\t\t\t\t\t\t",
1355 rwc_lf_results.w_no_ks_r_hit[j][i]);
1356 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1357 "%u\n\t\t\t\t\t\t\t\t",
1358 rwc_lf_results.w_no_ks_r_miss[j][i]);
1359 printf("Hash add - key-shifts, lookup - hit"
1360 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1361 rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1362 printf("Hash add - key-shifts, lookup - hit "
1363 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1364 rwc_lf_results.w_ks_r_hit_sp[j][i]);
1365 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1366 "%u\n\t\t\t\t\t\t\t\t",
1367 rwc_lf_results.w_ks_r_miss[j][i]);
1368 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1370 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1372 printf("Disabled\t");
1374 printf("Enabled\t\t");
1376 printf("Disabled\t");
1377 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1378 "%u\n\t\t\t\t\t\t\t\t",
1379 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1380 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1381 "%u\n\t\t\t\t\t\t\t\t",
1382 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1383 printf("Hash add - key-shifts, lookup - hit "
1384 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1385 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1386 printf("Hash add - key-shifts, lookup - hit "
1387 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1388 rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1389 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1390 "%u\n\t\t\t\t\t\t\t\t",
1391 rwc_non_lf_results.w_ks_r_miss[j][i]);
1392 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1394 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1396 printf("_______\t\t_______\t\t_________\t___\t\t"
1397 "_________\t\t\t\t\t\t_________________\n");
1400 for (i = 1; i < NUM_TEST; i++) {
1401 for (k = 0; k < NUM_TEST; k++) {
1402 printf("%u", rwc_core_cnt[i]);
1403 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1404 printf("Enabled\t\t");
1406 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1408 rwc_lf_results.multi_rw[i][j][k]);
1409 printf("Disabled\t");
1411 printf("Enabled\t\t");
1413 printf("Disabled\t");
1414 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1415 rwc_non_lf_results.multi_rw[i][j][k]);
1417 printf("_______\t\t_______\t\t_________\t___"
1418 "\t\t_________\t\t\t\t\t\t"
1419 "_________________\n");
1423 rte_free(tbl_rwc_test_param.keys);
1424 rte_free(tbl_rwc_test_param.keys_no_ks);
1425 rte_free(tbl_rwc_test_param.keys_ks);
1426 rte_free(tbl_rwc_test_param.keys_absent);
1427 rte_free(tbl_rwc_test_param.keys_shift_path);
1428 rte_free(tbl_rwc_test_param.keys_non_shift_path);
1429 rte_free(tbl_rwc_test_param.keys_ext_bkt);
1430 rte_free(tbl_rwc_test_param.keys_ks_extbkt);
1434 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);