1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Arm Limited
8 #include <rte_cycles.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
23 #define BULK_LOOKUP_SIZE 32
25 #define RUN_WITH_HTM_DISABLED 0
27 #if (RUN_WITH_HTM_DISABLED)
29 #define TOTAL_ENTRY (5*1024)
30 #define TOTAL_INSERT (5*1024)
34 #define TOTAL_ENTRY (4*1024*1024)
35 #define TOTAL_INSERT (4*1024*1024)
40 #define READ_PASS_NO_KEY_SHIFTS 2
41 #define READ_PASS_SHIFT_PATH 4
42 #define READ_PASS_NON_SHIFT_PATH 8
43 #define BULK_LOOKUP 16
44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
46 #define WRITE_NO_KEY_SHIFT 0
47 #define WRITE_KEY_SHIFT 1
48 #define WRITE_EXT_BKT 2
51 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
54 uint32_t w_no_ks_r_hit[2][NUM_TEST];
55 uint32_t w_no_ks_r_miss[2][NUM_TEST];
56 uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
57 uint32_t w_ks_r_hit_sp[2][NUM_TEST];
58 uint32_t w_ks_r_miss[2][NUM_TEST];
59 uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
60 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
69 uint32_t *keys_absent;
70 uint32_t *keys_shift_path;
71 uint32_t *keys_non_shift_path;
72 uint32_t *keys_ext_bkt;
73 uint32_t *keys_ks_extbkt;
74 uint32_t count_keys_no_ks;
75 uint32_t count_keys_ks;
76 uint32_t count_keys_absent;
77 uint32_t count_keys_shift_path;
78 uint32_t count_keys_non_shift_path;
79 uint32_t count_keys_extbkt;
80 uint32_t count_keys_ks_extbkt;
81 uint32_t single_insert;
85 static rte_atomic64_t gread_cycles;
86 static rte_atomic64_t greads;
88 static volatile uint8_t writer_done;
90 uint16_t enabled_core_ids[RTE_MAX_LCORE];
92 uint8_t *scanned_bkts;
94 static inline uint16_t
95 get_short_sig(const hash_sig_t hash)
100 static inline uint32_t
101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
102 const hash_sig_t hash)
104 uint32_t num_buckets;
105 uint32_t bucket_bitmask;
106 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
107 bucket_bitmask = num_buckets - 1;
108 return hash & bucket_bitmask;
111 static inline uint32_t
112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
113 uint32_t cur_bkt_idx, uint16_t sig)
115 uint32_t num_buckets;
116 uint32_t bucket_bitmask;
117 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
118 bucket_bitmask = num_buckets - 1;
119 return (cur_bkt_idx ^ sig) & bucket_bitmask;
124 get_enabled_cores_list(void)
128 uint32_t max_cores = rte_lcore_count();
129 for (core_id = 0; core_id < RTE_MAX_LCORE && i < max_cores; core_id++) {
130 if (rte_lcore_is_enabled(core_id)) {
131 enabled_core_ids[i] = core_id;
136 if (i != max_cores) {
137 printf("Number of enabled cores in list is different from "
138 "number given by rte_lcore_count()\n");
145 check_bucket(uint32_t bkt_idx, uint32_t key)
151 const void *next_key;
154 /* Temporary bucket to hold the keys */
155 uint32_t keys_in_bkt[8];
159 while (rte_hash_iterate(tbl_rwc_test_param.h,
160 &next_key, &next_data, &iter) >= 0) {
162 /* Check for duplicate entries */
163 if (*(const uint32_t *)next_key == key)
166 /* Identify if there is any free entry in the bucket */
167 diff = iter - prev_iter;
172 keys_in_bkt[count] = *(const uint32_t *)next_key;
175 /* All entries in the bucket are occupied */
179 * Check if bucket was not scanned before, to avoid
182 if (scanned_bkts[bkt_idx] == 0) {
184 * Since this bucket (pointed to by bkt_idx) is
185 * full, it is likely that key(s) in this
186 * bucket will be on the shift path, when
187 * collision occurs. Thus, add it to
190 memcpy(tbl_rwc_test_param.keys_shift_path +
191 tbl_rwc_test_param.count_keys_shift_path
193 tbl_rwc_test_param.count_keys_shift_path += 8;
194 scanned_bkts[bkt_idx] = 1;
205 uint32_t *keys = NULL;
206 uint32_t *keys_no_ks = NULL;
207 uint32_t *keys_ks = NULL;
208 uint32_t *keys_absent = NULL;
209 uint32_t *keys_non_shift_path = NULL;
210 uint32_t *keys_ext_bkt = NULL;
211 uint32_t *keys_ks_extbkt = NULL;
212 uint32_t *found = NULL;
213 uint32_t count_keys_no_ks = 0;
214 uint32_t count_keys_ks = 0;
215 uint32_t count_keys_extbkt = 0;
219 * keys will consist of a) keys whose addition to the hash table
220 * will result in shifting of the existing keys to their alternate
221 * locations b) keys whose addition to the hash table will not result
222 * in shifting of the existing keys.
224 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
226 printf("RTE_MALLOC failed\n");
231 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that
232 * will NOT result in shifting of the existing keys to their alternate
233 * locations. Roughly around 900K keys.
235 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
236 if (keys_no_ks == NULL) {
237 printf("RTE_MALLOC failed\n");
242 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
243 * result in shifting of the existing keys to their alternate locations.
244 * Roughly around 146K keys. There might be repeating keys. More code is
245 * required to filter out these keys which will complicate the test case
247 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
248 if (keys_ks == NULL) {
249 printf("RTE_MALLOC failed\n");
253 /* Used to identify keys not inserted in the hash table */
254 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
256 printf("RTE_MALLOC failed\n");
261 * This consist of keys not inserted to the hash table.
262 * Used to test perf of lookup on keys that do not exist in the table.
264 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
265 if (keys_absent == NULL) {
266 printf("RTE_MALLOC failed\n");
271 * This consist of keys which are likely to be on the shift
272 * path (i.e. being moved to alternate location), when collision occurs
273 * on addition of a key to an already full primary bucket.
274 * Used to test perf of lookup on keys that are on the shift path.
276 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
278 if (tbl_rwc_test_param.keys_shift_path == NULL) {
279 printf("RTE_MALLOC failed\n");
284 * This consist of keys which are never on the shift
285 * path (i.e. being moved to alternate location), when collision occurs
286 * on addition of a key to an already full primary bucket.
287 * Used to test perf of lookup on keys that are not on the shift path.
289 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
291 if (keys_non_shift_path == NULL) {
292 printf("RTE_MALLOC failed\n");
297 * This consist of keys which will be stored in extended buckets
299 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
300 if (keys_ext_bkt == NULL) {
301 printf("RTE_MALLOC failed\n");
306 * This consist of keys which when deleted causes shifting of keys
307 * in extended buckets to respective secondary buckets
309 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
310 if (keys_ks_extbkt == NULL) {
311 printf("RTE_MALLOC failed\n");
316 uint32_t prim_bucket_idx;
317 uint32_t sec_bucket_idx;
319 uint32_t num_buckets;
320 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
324 * Used to mark bkts in which at least one key was shifted to its
327 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
328 if (scanned_bkts == NULL) {
329 printf("RTE_MALLOC failed\n");
333 tbl_rwc_test_param.keys = keys;
334 tbl_rwc_test_param.keys_no_ks = keys_no_ks;
335 tbl_rwc_test_param.keys_ks = keys_ks;
336 tbl_rwc_test_param.keys_absent = keys_absent;
337 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
338 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
339 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
340 /* Generate keys by adding previous two keys, neglect overflow */
341 printf("Generating keys...\n");
344 for (i = 2; i < TOTAL_INSERT; i++)
345 keys[i] = keys[i-1] + keys[i-2];
347 /* Segregate keys into keys_no_ks and keys_ks */
348 for (i = 0; i < TOTAL_INSERT; i++) {
349 /* Check if primary bucket has space.*/
350 sig = rte_hash_hash(tbl_rwc_test_param.h,
351 tbl_rwc_test_param.keys+i);
352 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
354 ret = check_bucket(prim_bucket_idx, keys[i]);
357 * Primary bucket is full, this key will result in
358 * shifting of the keys to their alternate locations.
360 keys_ks[count_keys_ks] = keys[i];
362 } else if (ret == 0) {
364 * Primary bucket has space, this key will not result in
365 * shifting of the keys. Hence, add key to the table.
367 ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
369 (void *)((uintptr_t)i));
371 printf("writer failed %"PRIu32"\n", i);
374 keys_no_ks[count_keys_no_ks] = keys[i];
379 for (i = 0; i < count_keys_no_ks; i++) {
381 * Identify keys in keys_no_ks with value less than
382 * 4M (HTM enabled) OR 5K (HTM disabled)
384 if (keys_no_ks[i] < TOTAL_INSERT)
385 found[keys_no_ks[i]]++;
388 for (i = 0; i < count_keys_ks; i++) {
390 * Identify keys in keys_ks with value less than
391 * 4M (HTM enabled) OR 5K (HTM disabled)
393 if (keys_ks[i] < TOTAL_INSERT)
397 uint32_t count_keys_absent = 0;
398 for (i = 0; i < TOTAL_INSERT; i++) {
400 * Identify missing keys between 0 and
401 * 4M (HTM enabled) OR 5K (HTM disabled)
404 keys_absent[count_keys_absent++] = i;
407 /* Find keys that will not be on the shift path */
409 const void *next_key;
412 for (i = 0; i < num_buckets; i++) {
413 /* Check bucket for no keys shifted to alternate locations */
414 if (scanned_bkts[i] == 0) {
416 while (rte_hash_iterate(tbl_rwc_test_param.h,
417 &next_key, &next_data, &iter) >= 0) {
419 /* Check if key belongs to the current bucket */
421 keys_non_shift_path[count++]
422 = *(const uint32_t *)next_key;
429 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
430 tbl_rwc_test_param.count_keys_ks = count_keys_ks;
431 tbl_rwc_test_param.count_keys_absent = count_keys_absent;
432 tbl_rwc_test_param.count_keys_non_shift_path = count;
434 memset(scanned_bkts, 0, num_buckets);
436 /* Find keys that will be in extended buckets */
437 for (i = 0; i < count_keys_ks; i++) {
438 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
440 /* Key will be added to ext bkt */
441 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
442 /* Sec bkt to be added to keys_ks_extbkt */
443 sig = rte_hash_hash(tbl_rwc_test_param.h,
444 tbl_rwc_test_param.keys_ks + i);
445 prim_bucket_idx = get_prim_bucket_index(
446 tbl_rwc_test_param.h, sig);
447 short_sig = get_short_sig(sig);
448 sec_bucket_idx = get_alt_bucket_index(
449 tbl_rwc_test_param.h,
450 prim_bucket_idx, short_sig);
451 if (scanned_bkts[sec_bucket_idx] == 0)
452 scanned_bkts[sec_bucket_idx] = 1;
456 /* Find keys that will shift keys in ext bucket*/
457 for (i = 0; i < num_buckets; i++) {
458 if (scanned_bkts[i] == 1) {
460 while (rte_hash_iterate(tbl_rwc_test_param.h,
461 &next_key, &next_data, &iter) >= 0) {
462 /* Check if key belongs to the current bucket */
464 keys_ks_extbkt[count++]
465 = *(const uint32_t *)next_key;
472 tbl_rwc_test_param.count_keys_ks_extbkt = count;
473 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
475 printf("\nCount of keys NOT causing shifting of existing keys to "
476 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
477 printf("\nCount of keys causing shifting of existing keys to alternate "
478 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
479 printf("Count of absent keys that will never be added to the hash "
480 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
481 printf("Count of keys likely to be on the shift path: %d\n\n",
482 tbl_rwc_test_param.count_keys_shift_path);
483 printf("Count of keys not likely to be on the shift path: %d\n\n",
484 tbl_rwc_test_param.count_keys_non_shift_path);
485 printf("Count of keys in extended buckets: %d\n\n",
486 tbl_rwc_test_param.count_keys_extbkt);
487 printf("Count of keys shifting keys in ext buckets: %d\n\n",
488 tbl_rwc_test_param.count_keys_ks_extbkt);
491 rte_hash_free(tbl_rwc_test_param.h);
496 rte_free(keys_no_ks);
498 rte_free(keys_absent);
500 rte_free(tbl_rwc_test_param.keys_shift_path);
501 rte_free(keys_ext_bkt);
502 rte_free(keys_ks_extbkt);
503 rte_free(scanned_bkts);
508 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
510 struct rte_hash *handle;
512 struct rte_hash_parameters hash_params = {
513 .entries = TOTAL_ENTRY,
514 .key_len = sizeof(uint32_t),
515 .hash_func_init_val = 0,
516 .socket_id = rte_socket_id(),
520 hash_params.hash_func = rte_jhash;
522 hash_params.hash_func = rte_hash_crc;
525 hash_params.extra_flag =
526 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
527 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
529 hash_params.extra_flag =
530 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
531 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
532 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
534 hash_params.extra_flag =
535 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
536 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
539 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
541 hash_params.name = "tests";
543 handle = rte_hash_create(&hash_params);
544 if (handle == NULL) {
545 printf("hash creation failed");
549 tbl_rwc_test_param.h = handle;
554 test_rwc_reader(__attribute__((unused)) void *arg)
558 uint64_t begin, cycles;
559 uint32_t loop_cnt = 0;
560 uint8_t read_type = (uint8_t)((uintptr_t)arg);
565 void *temp_a[BULK_LOOKUP_SIZE];
567 /* Used to identify keys not inserted in the hash table */
568 pos = rte_malloc(NULL, sizeof(uint32_t) * BULK_LOOKUP_SIZE, 0);
570 printf("RTE_MALLOC failed\n");
574 if (read_type & READ_FAIL) {
575 keys = tbl_rwc_test_param.keys_absent;
576 read_cnt = tbl_rwc_test_param.count_keys_absent;
577 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
578 keys = tbl_rwc_test_param.keys_no_ks;
579 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
580 } else if (read_type & READ_PASS_SHIFT_PATH) {
581 keys = tbl_rwc_test_param.keys_shift_path;
582 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
583 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
584 keys = tbl_rwc_test_param.keys_ext_bkt;
585 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
587 keys = tbl_rwc_test_param.keys_non_shift_path;
588 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
591 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
593 begin = rte_rdtsc_precise();
595 if (read_type & BULK_LOOKUP) {
596 for (i = 0; i < (read_cnt - extra_keys);
597 i += BULK_LOOKUP_SIZE) {
598 /* Array of pointer to the list of keys */
599 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
600 temp_a[j] = keys + i + j;
601 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
604 BULK_LOOKUP_SIZE, pos);
605 /* Validate lookup result */
606 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
607 if ((read_type & READ_FAIL &&
608 pos[j] != -ENOENT) ||
609 (!(read_type & READ_FAIL) &&
610 pos[j] == -ENOENT)) {
611 printf("lookup failed!"
617 for (j = 0; j < extra_keys; j++)
618 temp_a[j] = keys + i + j;
620 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
624 for (j = 0; j < extra_keys; j++)
625 if ((read_type & READ_FAIL &&
626 pos[j] != -ENOENT) ||
627 (!(read_type & READ_FAIL) &&
628 pos[j] == -ENOENT)) {
629 printf("lookup failed! %"PRIu32"\n",
634 for (i = 0; i < read_cnt; i++) {
635 ret = rte_hash_lookup
636 (tbl_rwc_test_param.h, keys + i);
637 if (((read_type & READ_FAIL) &&
639 (!(read_type & READ_FAIL) &&
641 printf("lookup failed! %"PRIu32"\n",
648 } while (!writer_done);
650 cycles = rte_rdtsc_precise() - begin;
651 rte_atomic64_add(&gread_cycles, cycles);
652 rte_atomic64_add(&greads, read_cnt*loop_cnt);
657 write_keys(uint8_t write_type)
661 uint32_t key_cnt = 0;
663 if (write_type == WRITE_KEY_SHIFT) {
664 key_cnt = tbl_rwc_test_param.count_keys_ks;
665 keys = tbl_rwc_test_param.keys_ks;
666 } else if (write_type == WRITE_NO_KEY_SHIFT) {
667 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
668 keys = tbl_rwc_test_param.keys_no_ks;
669 } else if (write_type == WRITE_EXT_BKT) {
670 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
671 keys = tbl_rwc_test_param.keys_ext_bkt;
673 for (i = 0; i < key_cnt; i++) {
674 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
675 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
676 printf("writer failed %"PRIu32"\n", i);
684 test_rwc_multi_writer(__attribute__((unused)) void *arg)
687 uint32_t pos_core = (uint32_t)((uintptr_t)arg);
688 offset = pos_core * tbl_rwc_test_param.single_insert;
689 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
690 rte_hash_add_key(tbl_rwc_test_param.h,
691 tbl_rwc_test_param.keys_ks + i);
697 * Reader(s) lookup keys present in the table.
700 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
701 int htm, int ext_bkt)
706 uint8_t write_type = WRITE_NO_KEY_SHIFT;
707 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
709 rte_atomic64_init(&greads);
710 rte_atomic64_init(&gread_cycles);
712 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
714 printf("\nTest: Hash add - no key-shifts, read - hit\n");
715 for (m = 0; m < 2; m++) {
717 printf("\n** With bulk-lookup **\n");
718 read_type |= BULK_LOOKUP;
720 for (n = 0; n < NUM_TEST; n++) {
721 unsigned int tot_lcore = rte_lcore_count();
722 if (tot_lcore < rwc_core_cnt[n] + 1)
725 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
727 rte_atomic64_clear(&greads);
728 rte_atomic64_clear(&gread_cycles);
730 rte_hash_reset(tbl_rwc_test_param.h);
732 if (write_keys(write_type) < 0)
735 for (i = 1; i <= rwc_core_cnt[n]; i++)
736 rte_eal_remote_launch(test_rwc_reader,
737 (void *)(uintptr_t)read_type,
738 enabled_core_ids[i]);
740 for (i = 1; i <= rwc_core_cnt[n]; i++)
741 if (rte_eal_wait_lcore(i) < 0)
744 unsigned long long cycles_per_lookup =
745 rte_atomic64_read(&gread_cycles) /
746 rte_atomic64_read(&greads);
747 rwc_perf_results->w_no_ks_r_hit[m][n]
749 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
754 rte_hash_free(tbl_rwc_test_param.h);
758 rte_eal_mp_wait_lcore();
759 rte_hash_free(tbl_rwc_test_param.h);
765 * Reader(s) lookup keys absent in the table while
766 * 'Main' thread adds with no key-shifts.
769 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
770 int htm, int ext_bkt)
775 uint8_t write_type = WRITE_NO_KEY_SHIFT;
776 uint8_t read_type = READ_FAIL;
779 rte_atomic64_init(&greads);
780 rte_atomic64_init(&gread_cycles);
782 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
784 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
785 for (m = 0; m < 2; m++) {
787 printf("\n** With bulk-lookup **\n");
788 read_type |= BULK_LOOKUP;
790 for (n = 0; n < NUM_TEST; n++) {
791 unsigned int tot_lcore = rte_lcore_count();
792 if (tot_lcore < rwc_core_cnt[n] + 1)
795 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
797 rte_atomic64_clear(&greads);
798 rte_atomic64_clear(&gread_cycles);
800 rte_hash_reset(tbl_rwc_test_param.h);
803 for (i = 1; i <= rwc_core_cnt[n]; i++)
804 rte_eal_remote_launch(test_rwc_reader,
805 (void *)(uintptr_t)read_type,
806 enabled_core_ids[i]);
807 ret = write_keys(write_type);
812 for (i = 1; i <= rwc_core_cnt[n]; i++)
813 if (rte_eal_wait_lcore(i) < 0)
816 unsigned long long cycles_per_lookup =
817 rte_atomic64_read(&gread_cycles) /
818 rte_atomic64_read(&greads);
819 rwc_perf_results->w_no_ks_r_miss[m][n]
821 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
826 rte_hash_free(tbl_rwc_test_param.h);
830 rte_eal_mp_wait_lcore();
831 rte_hash_free(tbl_rwc_test_param.h);
837 * Reader(s) lookup keys present in the table and not likely to be on the
838 * shift path while 'Main' thread adds keys causing key-shifts.
841 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
842 int rwc_lf, int htm, int ext_bkt)
849 uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
851 rte_atomic64_init(&greads);
852 rte_atomic64_init(&gread_cycles);
854 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
856 printf("\nTest: Hash add - key shift, Hash lookup - hit"
857 " (non-shift-path)\n");
858 for (m = 0; m < 2; m++) {
860 printf("\n** With bulk-lookup **\n");
861 read_type |= BULK_LOOKUP;
863 for (n = 0; n < NUM_TEST; n++) {
864 unsigned int tot_lcore = rte_lcore_count();
865 if (tot_lcore < rwc_core_cnt[n] + 1)
868 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
870 rte_atomic64_clear(&greads);
871 rte_atomic64_clear(&gread_cycles);
873 rte_hash_reset(tbl_rwc_test_param.h);
875 write_type = WRITE_NO_KEY_SHIFT;
876 if (write_keys(write_type) < 0)
878 for (i = 1; i <= rwc_core_cnt[n]; i++)
879 rte_eal_remote_launch(test_rwc_reader,
880 (void *)(uintptr_t)read_type,
881 enabled_core_ids[i]);
882 write_type = WRITE_KEY_SHIFT;
883 ret = write_keys(write_type);
888 for (i = 1; i <= rwc_core_cnt[n]; i++)
889 if (rte_eal_wait_lcore(i) < 0)
892 unsigned long long cycles_per_lookup =
893 rte_atomic64_read(&gread_cycles) /
894 rte_atomic64_read(&greads);
895 rwc_perf_results->w_ks_r_hit_nsp[m][n]
897 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
902 rte_hash_free(tbl_rwc_test_param.h);
906 rte_eal_mp_wait_lcore();
907 rte_hash_free(tbl_rwc_test_param.h);
913 * Reader(s) lookup keys present in the table and likely on the shift-path while
914 * 'Main' thread adds keys causing key-shifts.
917 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
918 int htm, int ext_bkt)
925 uint8_t read_type = READ_PASS_SHIFT_PATH;
927 rte_atomic64_init(&greads);
928 rte_atomic64_init(&gread_cycles);
930 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
932 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
935 for (m = 0; m < 2; m++) {
937 printf("\n** With bulk-lookup **\n");
938 read_type |= BULK_LOOKUP;
940 for (n = 0; n < NUM_TEST; n++) {
941 unsigned int tot_lcore = rte_lcore_count();
942 if (tot_lcore < rwc_core_cnt[n] + 1)
945 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
946 rte_atomic64_clear(&greads);
947 rte_atomic64_clear(&gread_cycles);
949 rte_hash_reset(tbl_rwc_test_param.h);
951 write_type = WRITE_NO_KEY_SHIFT;
952 if (write_keys(write_type) < 0)
954 for (i = 1; i <= rwc_core_cnt[n]; i++)
955 rte_eal_remote_launch(test_rwc_reader,
956 (void *)(uintptr_t)read_type,
957 enabled_core_ids[i]);
958 write_type = WRITE_KEY_SHIFT;
959 ret = write_keys(write_type);
964 for (i = 1; i <= rwc_core_cnt[n]; i++)
965 if (rte_eal_wait_lcore(i) < 0)
968 unsigned long long cycles_per_lookup =
969 rte_atomic64_read(&gread_cycles) /
970 rte_atomic64_read(&greads);
971 rwc_perf_results->w_ks_r_hit_sp[m][n]
973 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
978 rte_hash_free(tbl_rwc_test_param.h);
982 rte_eal_mp_wait_lcore();
983 rte_hash_free(tbl_rwc_test_param.h);
989 * Reader(s) lookup keys absent in the table while
990 * 'Main' thread adds keys causing key-shifts.
993 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
1001 uint8_t read_type = READ_FAIL;
1003 rte_atomic64_init(&greads);
1004 rte_atomic64_init(&gread_cycles);
1006 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1008 printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1009 for (m = 0; m < 2; m++) {
1011 printf("\n** With bulk-lookup **\n");
1012 read_type |= BULK_LOOKUP;
1014 for (n = 0; n < NUM_TEST; n++) {
1015 unsigned int tot_lcore = rte_lcore_count();
1016 if (tot_lcore < rwc_core_cnt[n] + 1)
1019 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1021 rte_atomic64_clear(&greads);
1022 rte_atomic64_clear(&gread_cycles);
1024 rte_hash_reset(tbl_rwc_test_param.h);
1026 write_type = WRITE_NO_KEY_SHIFT;
1027 if (write_keys(write_type) < 0)
1029 for (i = 1; i <= rwc_core_cnt[n]; i++)
1030 rte_eal_remote_launch(test_rwc_reader,
1031 (void *)(uintptr_t)read_type,
1032 enabled_core_ids[i]);
1033 write_type = WRITE_KEY_SHIFT;
1034 ret = write_keys(write_type);
1039 for (i = 1; i <= rwc_core_cnt[n]; i++)
1040 if (rte_eal_wait_lcore(i) < 0)
1043 unsigned long long cycles_per_lookup =
1044 rte_atomic64_read(&gread_cycles) /
1045 rte_atomic64_read(&greads);
1046 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1047 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1052 rte_hash_free(tbl_rwc_test_param.h);
1056 rte_eal_mp_wait_lcore();
1057 rte_hash_free(tbl_rwc_test_param.h);
1062 * Test lookup perf for multi-writer:
1063 * Reader(s) lookup keys present in the table and likely on the shift-path while
1064 * Writers add keys causing key-shiftsi.
1065 * Writers are running in parallel, on different data plane cores.
1068 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1069 int htm, int ext_bkt)
1071 unsigned int n, m, k;
1075 uint8_t read_type = READ_PASS_SHIFT_PATH;
1077 rte_atomic64_init(&greads);
1078 rte_atomic64_init(&gread_cycles);
1080 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1082 printf("\nTest: Multi-add-lookup\n");
1084 for (m = 1; m < NUM_TEST; m++) {
1085 /* Calculate keys added by each writer */
1086 tbl_rwc_test_param.single_insert =
1087 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1088 for (k = 0; k < 2; k++) {
1090 printf("\n** With bulk-lookup **\n");
1091 read_type |= BULK_LOOKUP;
1093 for (n = 0; n < NUM_TEST; n++) {
1094 unsigned int tot_lcore = rte_lcore_count();
1095 if (tot_lcore < (rwc_core_cnt[n] +
1096 rwc_core_cnt[m] + 1))
1099 printf("\nNumber of writers: %u",
1101 printf("\nNumber of readers: %u\n",
1104 rte_atomic64_clear(&greads);
1105 rte_atomic64_clear(&gread_cycles);
1107 rte_hash_reset(tbl_rwc_test_param.h);
1109 write_type = WRITE_NO_KEY_SHIFT;
1110 if (write_keys(write_type) < 0)
1113 /* Launch reader(s) */
1114 for (i = 1; i <= rwc_core_cnt[n]; i++)
1115 rte_eal_remote_launch(test_rwc_reader,
1116 (void *)(uintptr_t)read_type,
1117 enabled_core_ids[i]);
1118 write_type = WRITE_KEY_SHIFT;
1121 /* Launch writers */
1122 for (; i <= rwc_core_cnt[m]
1123 + rwc_core_cnt[n]; i++) {
1124 rte_eal_remote_launch
1125 (test_rwc_multi_writer,
1126 (void *)(uintptr_t)pos_core,
1127 enabled_core_ids[i]);
1131 /* Wait for writers to complete */
1132 for (i = rwc_core_cnt[n] + 1;
1133 i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1135 rte_eal_wait_lcore(i);
1139 for (i = 1; i <= rwc_core_cnt[n]; i++)
1140 if (rte_eal_wait_lcore(i) < 0)
1143 unsigned long long cycles_per_lookup =
1144 rte_atomic64_read(&gread_cycles)
1145 / rte_atomic64_read(&greads);
1146 rwc_perf_results->multi_rw[m][k][n]
1147 = cycles_per_lookup;
1148 printf("Cycles per lookup: %llu\n",
1155 rte_hash_free(tbl_rwc_test_param.h);
1159 rte_eal_mp_wait_lcore();
1160 rte_hash_free(tbl_rwc_test_param.h);
1166 * Reader(s) lookup keys present in the extendable bkt.
1169 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1170 int rwc_lf, int htm, int ext_bkt)
1176 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1178 rte_atomic64_init(&greads);
1179 rte_atomic64_init(&gread_cycles);
1181 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1183 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1184 for (m = 0; m < 2; m++) {
1186 printf("\n** With bulk-lookup **\n");
1187 read_type |= BULK_LOOKUP;
1189 for (n = 0; n < NUM_TEST; n++) {
1190 unsigned int tot_lcore = rte_lcore_count();
1191 if (tot_lcore < rwc_core_cnt[n] + 1)
1194 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1196 rte_atomic64_clear(&greads);
1197 rte_atomic64_clear(&gread_cycles);
1199 rte_hash_reset(tbl_rwc_test_param.h);
1200 write_type = WRITE_NO_KEY_SHIFT;
1201 if (write_keys(write_type) < 0)
1203 write_type = WRITE_KEY_SHIFT;
1204 if (write_keys(write_type) < 0)
1207 for (i = 1; i <= rwc_core_cnt[n]; i++)
1208 rte_eal_remote_launch(test_rwc_reader,
1209 (void *)(uintptr_t)read_type,
1210 enabled_core_ids[i]);
1211 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1213 if (rte_hash_del_key(tbl_rwc_test_param.h,
1214 tbl_rwc_test_param.keys_ks_extbkt + i)
1216 printf("Delete Failed: %u\n",
1217 tbl_rwc_test_param.keys_ks_extbkt[i]);
1223 for (i = 1; i <= rwc_core_cnt[n]; i++)
1224 if (rte_eal_wait_lcore(i) < 0)
1227 unsigned long long cycles_per_lookup =
1228 rte_atomic64_read(&gread_cycles) /
1229 rte_atomic64_read(&greads);
1230 rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1231 = cycles_per_lookup;
1232 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1237 rte_hash_free(tbl_rwc_test_param.h);
1241 rte_eal_mp_wait_lcore();
1242 rte_hash_free(tbl_rwc_test_param.h);
1247 test_hash_readwrite_lf_main(void)
1250 * Variables used to choose different tests.
1251 * rwc_lf indicates if read-write concurrency lock-free support is
1253 * htm indicates if Hardware transactional memory support is enabled.
1259 if (rte_lcore_count() == 1) {
1260 printf("More than one lcore is required "
1261 "to do read write lock-free concurrency test\n");
1265 setlocale(LC_NUMERIC, "");
1267 if (rte_tm_supported())
1272 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1274 if (generate_keys() != 0)
1276 if (get_enabled_cores_list() != 0)
1279 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1282 printf("Test lookup with read-write concurrency lock free support"
1284 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1287 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1290 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1293 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1296 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1299 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1302 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1306 printf("\nTest lookup with read-write concurrency lock free support"
1310 printf("With HTM Disabled\n");
1311 if (!RUN_WITH_HTM_DISABLED) {
1312 printf("Enable RUN_WITH_HTM_DISABLED to test with"
1313 " lock-free disabled");
1317 printf("With HTM Enabled\n");
1318 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1321 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1324 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1327 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1330 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1333 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1336 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1340 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1342 for (j = 0; j < 2; j++) {
1344 printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1345 "**********#######\n\n");
1346 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1347 "\t\t\t\t_________________\n");
1348 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1349 "\t\t\tCycles per lookup\n");
1350 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1351 "\t\t\t_________________\n");
1352 for (i = 0; i < NUM_TEST; i++) {
1353 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1354 printf("Enabled\t\t");
1356 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1357 "%u\n\t\t\t\t\t\t\t\t",
1358 rwc_lf_results.w_no_ks_r_hit[j][i]);
1359 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1360 "%u\n\t\t\t\t\t\t\t\t",
1361 rwc_lf_results.w_no_ks_r_miss[j][i]);
1362 printf("Hash add - key-shifts, lookup - hit"
1363 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1364 rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1365 printf("Hash add - key-shifts, lookup - hit "
1366 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1367 rwc_lf_results.w_ks_r_hit_sp[j][i]);
1368 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1369 "%u\n\t\t\t\t\t\t\t\t",
1370 rwc_lf_results.w_ks_r_miss[j][i]);
1371 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1373 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1375 printf("Disabled\t");
1377 printf("Enabled\t\t");
1379 printf("Disabled\t");
1380 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1381 "%u\n\t\t\t\t\t\t\t\t",
1382 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1383 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1384 "%u\n\t\t\t\t\t\t\t\t",
1385 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1386 printf("Hash add - key-shifts, lookup - hit "
1387 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1388 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1389 printf("Hash add - key-shifts, lookup - hit "
1390 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1391 rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1392 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1393 "%u\n\t\t\t\t\t\t\t\t",
1394 rwc_non_lf_results.w_ks_r_miss[j][i]);
1395 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1397 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1399 printf("_______\t\t_______\t\t_________\t___\t\t"
1400 "_________\t\t\t\t\t\t_________________\n");
1403 for (i = 1; i < NUM_TEST; i++) {
1404 for (k = 0; k < NUM_TEST; k++) {
1405 printf("%u", rwc_core_cnt[i]);
1406 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1407 printf("Enabled\t\t");
1409 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1411 rwc_lf_results.multi_rw[i][j][k]);
1412 printf("Disabled\t");
1414 printf("Enabled\t\t");
1416 printf("Disabled\t");
1417 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1418 rwc_non_lf_results.multi_rw[i][j][k]);
1420 printf("_______\t\t_______\t\t_________\t___"
1421 "\t\t_________\t\t\t\t\t\t"
1422 "_________________\n");
1426 rte_free(tbl_rwc_test_param.keys);
1427 rte_free(tbl_rwc_test_param.keys_no_ks);
1428 rte_free(tbl_rwc_test_param.keys_ks);
1429 rte_free(tbl_rwc_test_param.keys_absent);
1430 rte_free(tbl_rwc_test_param.keys_shift_path);
1431 rte_free(scanned_bkts);
1435 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);