1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Arm Limited
8 #include <rte_cycles.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
23 #define BULK_LOOKUP_SIZE 32
25 #define RUN_WITH_HTM_DISABLED 0
27 #if (RUN_WITH_HTM_DISABLED)
29 #define TOTAL_ENTRY (5*1024)
30 #define TOTAL_INSERT (5*1024)
34 #define TOTAL_ENTRY (4*1024*1024)
35 #define TOTAL_INSERT (4*1024*1024)
40 #define READ_PASS_NO_KEY_SHIFTS 2
41 #define READ_PASS_SHIFT_PATH 4
42 #define READ_PASS_NON_SHIFT_PATH 8
43 #define BULK_LOOKUP 16
44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
46 #define WRITE_NO_KEY_SHIFT 0
47 #define WRITE_KEY_SHIFT 1
48 #define WRITE_EXT_BKT 2
52 #define QSBR_REPORTING_INTERVAL 1024
54 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
57 uint32_t w_no_ks_r_hit[2][NUM_TEST];
58 uint32_t w_no_ks_r_miss[2][NUM_TEST];
59 uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
60 uint32_t w_ks_r_hit_sp[2][NUM_TEST];
61 uint32_t w_ks_r_miss[2][NUM_TEST];
62 uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
63 uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
64 uint32_t writer_add_del[NUM_TEST];
67 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
73 uint32_t *keys_absent;
74 uint32_t *keys_shift_path;
75 uint32_t *keys_non_shift_path;
76 uint32_t *keys_ext_bkt;
77 uint32_t *keys_ks_extbkt;
78 uint32_t count_keys_no_ks;
79 uint32_t count_keys_ks;
80 uint32_t count_keys_absent;
81 uint32_t count_keys_shift_path;
82 uint32_t count_keys_non_shift_path;
83 uint32_t count_keys_extbkt;
84 uint32_t count_keys_ks_extbkt;
85 uint32_t single_insert;
89 static uint64_t gread_cycles;
90 static uint64_t greads;
91 static uint64_t gwrite_cycles;
92 static uint64_t gwrites;
94 static volatile uint8_t writer_done;
96 static uint16_t enabled_core_ids[RTE_MAX_LCORE];
98 static uint8_t *scanned_bkts;
100 static inline uint16_t
101 get_short_sig(const hash_sig_t hash)
106 static inline uint32_t
107 get_prim_bucket_index(__rte_unused const struct rte_hash *h,
108 const hash_sig_t hash)
110 uint32_t num_buckets;
111 uint32_t bucket_bitmask;
112 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
113 bucket_bitmask = num_buckets - 1;
114 return hash & bucket_bitmask;
117 static inline uint32_t
118 get_alt_bucket_index(__rte_unused const struct rte_hash *h,
119 uint32_t cur_bkt_idx, uint16_t sig)
121 uint32_t num_buckets;
122 uint32_t bucket_bitmask;
123 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
124 bucket_bitmask = num_buckets - 1;
125 return (cur_bkt_idx ^ sig) & bucket_bitmask;
130 get_enabled_cores_list(void)
134 uint32_t max_cores = rte_lcore_count();
135 RTE_LCORE_FOREACH(core_id) {
136 enabled_core_ids[i] = core_id;
140 if (i != max_cores) {
141 printf("Number of enabled cores in list is different from "
142 "number given by rte_lcore_count()\n");
149 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
151 struct rte_hash *handle;
153 struct rte_hash_parameters hash_params = {
154 .entries = TOTAL_ENTRY,
155 .key_len = sizeof(uint32_t),
156 .hash_func_init_val = 0,
157 .socket_id = rte_socket_id(),
161 hash_params.hash_func = rte_jhash;
163 hash_params.hash_func = rte_hash_crc;
166 hash_params.extra_flag =
167 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
168 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
170 hash_params.extra_flag =
171 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
172 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
173 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
175 hash_params.extra_flag =
176 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
177 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
180 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
182 hash_params.name = "tests";
184 handle = rte_hash_create(&hash_params);
185 if (handle == NULL) {
186 printf("hash creation failed");
190 tbl_rwc_test_param.h = handle;
195 check_bucket(uint32_t bkt_idx, uint32_t key)
201 const void *next_key;
204 /* Temporary bucket to hold the keys */
205 uint32_t keys_in_bkt[8];
209 while (rte_hash_iterate(tbl_rwc_test_param.h,
210 &next_key, &next_data, &iter) >= 0) {
212 /* Check for duplicate entries */
213 if (*(const uint32_t *)next_key == key)
216 /* Identify if there is any free entry in the bucket */
217 diff = iter - prev_iter;
222 keys_in_bkt[count] = *(const uint32_t *)next_key;
225 /* All entries in the bucket are occupied */
229 * Check if bucket was not scanned before, to avoid
232 if (scanned_bkts[bkt_idx] == 0) {
234 * Since this bucket (pointed to by bkt_idx) is
235 * full, it is likely that key(s) in this
236 * bucket will be on the shift path, when
237 * collision occurs. Thus, add it to
240 memcpy(tbl_rwc_test_param.keys_shift_path +
241 tbl_rwc_test_param.count_keys_shift_path
243 tbl_rwc_test_param.count_keys_shift_path += 8;
244 scanned_bkts[bkt_idx] = 1;
255 uint32_t *keys = NULL;
256 uint32_t *keys_no_ks = NULL;
257 uint32_t *keys_ks = NULL;
258 uint32_t *keys_absent = NULL;
259 uint32_t *keys_non_shift_path = NULL;
260 uint32_t *keys_ext_bkt = NULL;
261 uint32_t *keys_ks_extbkt = NULL;
262 uint32_t *found = NULL;
263 uint32_t count_keys_no_ks = 0;
264 uint32_t count_keys_ks = 0;
265 uint32_t count_keys_extbkt = 0;
268 if (init_params(0, 0, 0, 0) != 0)
272 * keys will consist of a) keys whose addition to the hash table
273 * will result in shifting of the existing keys to their alternate
274 * locations b) keys whose addition to the hash table will not result
275 * in shifting of the existing keys.
277 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
279 printf("RTE_MALLOC failed\n");
284 * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that
285 * will NOT result in shifting of the existing keys to their alternate
286 * locations. Roughly around 900K keys.
288 keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
289 if (keys_no_ks == NULL) {
290 printf("RTE_MALLOC failed\n");
295 * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
296 * result in shifting of the existing keys to their alternate locations.
297 * Roughly around 146K keys. There might be repeating keys. More code is
298 * required to filter out these keys which will complicate the test case
300 keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
301 if (keys_ks == NULL) {
302 printf("RTE_MALLOC failed\n");
306 /* Used to identify keys not inserted in the hash table */
307 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
309 printf("RTE_MALLOC failed\n");
314 * This consist of keys not inserted to the hash table.
315 * Used to test perf of lookup on keys that do not exist in the table.
317 keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
318 if (keys_absent == NULL) {
319 printf("RTE_MALLOC failed\n");
324 * This consist of keys which are likely to be on the shift
325 * path (i.e. being moved to alternate location), when collision occurs
326 * on addition of a key to an already full primary bucket.
327 * Used to test perf of lookup on keys that are on the shift path.
329 tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
331 if (tbl_rwc_test_param.keys_shift_path == NULL) {
332 printf("RTE_MALLOC failed\n");
337 * This consist of keys which are never on the shift
338 * path (i.e. being moved to alternate location), when collision occurs
339 * on addition of a key to an already full primary bucket.
340 * Used to test perf of lookup on keys that are not on the shift path.
342 keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
344 if (keys_non_shift_path == NULL) {
345 printf("RTE_MALLOC failed\n");
350 * This consist of keys which will be stored in extended buckets
352 keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
353 if (keys_ext_bkt == NULL) {
354 printf("RTE_MALLOC failed\n");
359 * This consist of keys which when deleted causes shifting of keys
360 * in extended buckets to respective secondary buckets
362 keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
363 if (keys_ks_extbkt == NULL) {
364 printf("RTE_MALLOC failed\n");
369 uint32_t prim_bucket_idx;
370 uint32_t sec_bucket_idx;
372 uint32_t num_buckets;
373 num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8;
377 * Used to mark bkts in which at least one key was shifted to its
380 scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
381 if (scanned_bkts == NULL) {
382 printf("RTE_MALLOC failed\n");
386 tbl_rwc_test_param.keys = keys;
387 tbl_rwc_test_param.keys_no_ks = keys_no_ks;
388 tbl_rwc_test_param.keys_ks = keys_ks;
389 tbl_rwc_test_param.keys_absent = keys_absent;
390 tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
391 tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
392 tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
393 /* Generate keys by adding previous two keys, neglect overflow */
394 printf("Generating keys...\n");
397 for (i = 2; i < TOTAL_INSERT; i++)
398 keys[i] = keys[i-1] + keys[i-2];
400 /* Segregate keys into keys_no_ks and keys_ks */
401 for (i = 0; i < TOTAL_INSERT; i++) {
402 /* Check if primary bucket has space.*/
403 sig = rte_hash_hash(tbl_rwc_test_param.h,
404 tbl_rwc_test_param.keys+i);
405 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
407 ret = check_bucket(prim_bucket_idx, keys[i]);
410 * Primary bucket is full, this key will result in
411 * shifting of the keys to their alternate locations.
413 keys_ks[count_keys_ks] = keys[i];
415 } else if (ret == 0) {
417 * Primary bucket has space, this key will not result in
418 * shifting of the keys. Hence, add key to the table.
420 ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
422 (void *)((uintptr_t)i));
424 printf("writer failed %"PRIu32"\n", i);
427 keys_no_ks[count_keys_no_ks] = keys[i];
432 for (i = 0; i < count_keys_no_ks; i++) {
434 * Identify keys in keys_no_ks with value less than
435 * 4M (HTM enabled) OR 5K (HTM disabled)
437 if (keys_no_ks[i] < TOTAL_INSERT)
438 found[keys_no_ks[i]]++;
441 for (i = 0; i < count_keys_ks; i++) {
443 * Identify keys in keys_ks with value less than
444 * 4M (HTM enabled) OR 5K (HTM disabled)
446 if (keys_ks[i] < TOTAL_INSERT)
450 uint32_t count_keys_absent = 0;
451 for (i = 0; i < TOTAL_INSERT; i++) {
453 * Identify missing keys between 0 and
454 * 4M (HTM enabled) OR 5K (HTM disabled)
457 keys_absent[count_keys_absent++] = i;
460 /* Find keys that will not be on the shift path */
462 const void *next_key;
465 for (i = 0; i < num_buckets; i++) {
466 /* Check bucket for no keys shifted to alternate locations */
467 if (scanned_bkts[i] == 0) {
469 while (rte_hash_iterate(tbl_rwc_test_param.h,
470 &next_key, &next_data, &iter) >= 0) {
472 /* Check if key belongs to the current bucket */
474 keys_non_shift_path[count++]
475 = *(const uint32_t *)next_key;
482 tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
483 tbl_rwc_test_param.count_keys_ks = count_keys_ks;
484 tbl_rwc_test_param.count_keys_absent = count_keys_absent;
485 tbl_rwc_test_param.count_keys_non_shift_path = count;
487 memset(scanned_bkts, 0, num_buckets);
489 /* Find keys that will be in extended buckets */
490 for (i = 0; i < count_keys_ks; i++) {
491 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
493 /* Key will be added to ext bkt */
494 keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
495 /* Sec bkt to be added to keys_ks_extbkt */
496 sig = rte_hash_hash(tbl_rwc_test_param.h,
497 tbl_rwc_test_param.keys_ks + i);
498 prim_bucket_idx = get_prim_bucket_index(
499 tbl_rwc_test_param.h, sig);
500 short_sig = get_short_sig(sig);
501 sec_bucket_idx = get_alt_bucket_index(
502 tbl_rwc_test_param.h,
503 prim_bucket_idx, short_sig);
504 if (scanned_bkts[sec_bucket_idx] == 0)
505 scanned_bkts[sec_bucket_idx] = 1;
509 /* Find keys that will shift keys in ext bucket*/
510 for (i = 0; i < num_buckets; i++) {
511 if (scanned_bkts[i] == 1) {
513 while (rte_hash_iterate(tbl_rwc_test_param.h,
514 &next_key, &next_data, &iter) >= 0) {
515 /* Check if key belongs to the current bucket */
517 keys_ks_extbkt[count++]
518 = *(const uint32_t *)next_key;
525 tbl_rwc_test_param.count_keys_ks_extbkt = count;
526 tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
528 printf("\nCount of keys NOT causing shifting of existing keys to "
529 "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
530 printf("\nCount of keys causing shifting of existing keys to alternate "
531 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
532 printf("Count of absent keys that will never be added to the hash "
533 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
534 printf("Count of keys likely to be on the shift path: %d\n\n",
535 tbl_rwc_test_param.count_keys_shift_path);
536 printf("Count of keys not likely to be on the shift path: %d\n\n",
537 tbl_rwc_test_param.count_keys_non_shift_path);
538 printf("Count of keys in extended buckets: %d\n\n",
539 tbl_rwc_test_param.count_keys_extbkt);
540 printf("Count of keys shifting keys in ext buckets: %d\n\n",
541 tbl_rwc_test_param.count_keys_ks_extbkt);
544 rte_free(scanned_bkts);
545 rte_hash_free(tbl_rwc_test_param.h);
550 rte_free(keys_no_ks);
552 rte_free(keys_absent);
554 rte_free(tbl_rwc_test_param.keys_shift_path);
555 rte_free(keys_non_shift_path);
556 rte_free(keys_ext_bkt);
557 rte_free(keys_ks_extbkt);
558 rte_free(scanned_bkts);
559 rte_hash_free(tbl_rwc_test_param.h);
564 test_rwc_reader(__rte_unused void *arg)
568 uint64_t begin, cycles;
569 uint32_t loop_cnt = 0;
570 uint8_t read_type = (uint8_t)((uintptr_t)arg);
574 int32_t pos[BULK_LOOKUP_SIZE];
575 void *temp_a[BULK_LOOKUP_SIZE];
577 if (read_type & READ_FAIL) {
578 keys = tbl_rwc_test_param.keys_absent;
579 read_cnt = tbl_rwc_test_param.count_keys_absent;
580 } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
581 keys = tbl_rwc_test_param.keys_no_ks;
582 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
583 } else if (read_type & READ_PASS_SHIFT_PATH) {
584 keys = tbl_rwc_test_param.keys_shift_path;
585 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
586 } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
587 keys = tbl_rwc_test_param.keys_ext_bkt;
588 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
590 keys = tbl_rwc_test_param.keys_non_shift_path;
591 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
594 extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
596 begin = rte_rdtsc_precise();
598 if (read_type & BULK_LOOKUP) {
599 for (i = 0; i < (read_cnt - extra_keys);
600 i += BULK_LOOKUP_SIZE) {
601 /* Array of pointer to the list of keys */
602 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
603 temp_a[j] = keys + i + j;
604 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
607 BULK_LOOKUP_SIZE, pos);
608 /* Validate lookup result */
609 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
610 if ((read_type & READ_FAIL &&
611 pos[j] != -ENOENT) ||
612 (!(read_type & READ_FAIL) &&
613 pos[j] == -ENOENT)) {
614 printf("lookup failed!"
620 for (j = 0; j < extra_keys; j++)
621 temp_a[j] = keys + i + j;
623 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
627 for (j = 0; j < extra_keys; j++)
628 if ((read_type & READ_FAIL &&
629 pos[j] != -ENOENT) ||
630 (!(read_type & READ_FAIL) &&
631 pos[j] == -ENOENT)) {
632 printf("lookup failed! %"PRIu32"\n",
637 for (i = 0; i < read_cnt; i++) {
638 ret = rte_hash_lookup
639 (tbl_rwc_test_param.h, keys + i);
640 if (((read_type & READ_FAIL) &&
642 (!(read_type & READ_FAIL) &&
644 printf("lookup failed! %"PRIu32"\n",
651 } while (!writer_done);
653 cycles = rte_rdtsc_precise() - begin;
654 __atomic_fetch_add(&gread_cycles, cycles, __ATOMIC_RELAXED);
655 __atomic_fetch_add(&greads, read_cnt*loop_cnt, __ATOMIC_RELAXED);
660 write_keys(uint8_t write_type)
664 uint32_t key_cnt = 0;
666 if (write_type == WRITE_KEY_SHIFT) {
667 key_cnt = tbl_rwc_test_param.count_keys_ks;
668 keys = tbl_rwc_test_param.keys_ks;
669 } else if (write_type == WRITE_NO_KEY_SHIFT) {
670 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
671 keys = tbl_rwc_test_param.keys_no_ks;
672 } else if (write_type == WRITE_EXT_BKT) {
673 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
674 keys = tbl_rwc_test_param.keys_ext_bkt;
676 for (i = 0; i < key_cnt; i++) {
677 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
678 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
679 printf("writer failed %"PRIu32"\n", i);
687 test_rwc_multi_writer(__rte_unused void *arg)
690 uint32_t pos_core = (uint32_t)((uintptr_t)arg);
691 offset = pos_core * tbl_rwc_test_param.single_insert;
692 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
693 rte_hash_add_key(tbl_rwc_test_param.h,
694 tbl_rwc_test_param.keys_ks + i);
700 * Reader(s) lookup keys present in the table.
703 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
704 int htm, int ext_bkt)
709 uint8_t write_type = WRITE_NO_KEY_SHIFT;
710 uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
712 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
714 printf("\nTest: Hash add - no key-shifts, read - hit\n");
715 for (m = 0; m < 2; m++) {
717 printf("\n** With bulk-lookup **\n");
718 read_type |= BULK_LOOKUP;
720 for (n = 0; n < NUM_TEST; n++) {
721 unsigned int tot_lcore = rte_lcore_count();
722 if (tot_lcore < rwc_core_cnt[n] + 1)
725 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
727 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
728 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
730 rte_hash_reset(tbl_rwc_test_param.h);
732 if (write_keys(write_type) < 0)
735 for (i = 1; i <= rwc_core_cnt[n]; i++)
736 rte_eal_remote_launch(test_rwc_reader,
737 (void *)(uintptr_t)read_type,
738 enabled_core_ids[i]);
740 for (i = 1; i <= rwc_core_cnt[n]; i++)
741 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
744 unsigned long long cycles_per_lookup =
745 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
746 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
747 rwc_perf_results->w_no_ks_r_hit[m][n]
749 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
754 rte_hash_free(tbl_rwc_test_param.h);
758 rte_eal_mp_wait_lcore();
759 rte_hash_free(tbl_rwc_test_param.h);
765 * Reader(s) lookup keys absent in the table while
766 * 'Main' thread adds with no key-shifts.
769 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
770 int htm, int ext_bkt)
775 uint8_t write_type = WRITE_NO_KEY_SHIFT;
776 uint8_t read_type = READ_FAIL;
779 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
781 printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
782 for (m = 0; m < 2; m++) {
784 printf("\n** With bulk-lookup **\n");
785 read_type |= BULK_LOOKUP;
787 for (n = 0; n < NUM_TEST; n++) {
788 unsigned int tot_lcore = rte_lcore_count();
789 if (tot_lcore < rwc_core_cnt[n] + 1)
792 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
794 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
795 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
797 rte_hash_reset(tbl_rwc_test_param.h);
800 for (i = 1; i <= rwc_core_cnt[n]; i++)
801 rte_eal_remote_launch(test_rwc_reader,
802 (void *)(uintptr_t)read_type,
803 enabled_core_ids[i]);
804 ret = write_keys(write_type);
809 for (i = 1; i <= rwc_core_cnt[n]; i++)
810 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
813 unsigned long long cycles_per_lookup =
814 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
815 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
816 rwc_perf_results->w_no_ks_r_miss[m][n]
818 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
823 rte_hash_free(tbl_rwc_test_param.h);
827 rte_eal_mp_wait_lcore();
828 rte_hash_free(tbl_rwc_test_param.h);
834 * Reader(s) lookup keys present in the table and not likely to be on the
835 * shift path while 'Main' thread adds keys causing key-shifts.
838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
839 int rwc_lf, int htm, int ext_bkt)
846 uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
848 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
850 printf("\nTest: Hash add - key shift, Hash lookup - hit"
851 " (non-shift-path)\n");
852 for (m = 0; m < 2; m++) {
854 printf("\n** With bulk-lookup **\n");
855 read_type |= BULK_LOOKUP;
857 for (n = 0; n < NUM_TEST; n++) {
858 unsigned int tot_lcore = rte_lcore_count();
859 if (tot_lcore < rwc_core_cnt[n] + 1)
862 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
864 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
865 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
867 rte_hash_reset(tbl_rwc_test_param.h);
869 write_type = WRITE_NO_KEY_SHIFT;
870 if (write_keys(write_type) < 0)
872 for (i = 1; i <= rwc_core_cnt[n]; i++)
873 rte_eal_remote_launch(test_rwc_reader,
874 (void *)(uintptr_t)read_type,
875 enabled_core_ids[i]);
876 write_type = WRITE_KEY_SHIFT;
877 ret = write_keys(write_type);
882 for (i = 1; i <= rwc_core_cnt[n]; i++)
883 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
886 unsigned long long cycles_per_lookup =
887 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
888 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
889 rwc_perf_results->w_ks_r_hit_nsp[m][n]
891 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
896 rte_hash_free(tbl_rwc_test_param.h);
900 rte_eal_mp_wait_lcore();
901 rte_hash_free(tbl_rwc_test_param.h);
907 * Reader(s) lookup keys present in the table and likely on the shift-path while
908 * 'Main' thread adds keys causing key-shifts.
911 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
912 int htm, int ext_bkt)
919 uint8_t read_type = READ_PASS_SHIFT_PATH;
921 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
923 printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
926 for (m = 0; m < 2; m++) {
928 printf("\n** With bulk-lookup **\n");
929 read_type |= BULK_LOOKUP;
931 for (n = 0; n < NUM_TEST; n++) {
932 unsigned int tot_lcore = rte_lcore_count();
933 if (tot_lcore < rwc_core_cnt[n] + 1)
936 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
938 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
939 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
941 rte_hash_reset(tbl_rwc_test_param.h);
943 write_type = WRITE_NO_KEY_SHIFT;
944 if (write_keys(write_type) < 0)
946 for (i = 1; i <= rwc_core_cnt[n]; i++)
947 rte_eal_remote_launch(test_rwc_reader,
948 (void *)(uintptr_t)read_type,
949 enabled_core_ids[i]);
950 write_type = WRITE_KEY_SHIFT;
951 ret = write_keys(write_type);
956 for (i = 1; i <= rwc_core_cnt[n]; i++)
957 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
960 unsigned long long cycles_per_lookup =
961 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
962 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
963 rwc_perf_results->w_ks_r_hit_sp[m][n]
965 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
970 rte_hash_free(tbl_rwc_test_param.h);
974 rte_eal_mp_wait_lcore();
975 rte_hash_free(tbl_rwc_test_param.h);
981 * Reader(s) lookup keys absent in the table while
982 * 'Main' thread adds keys causing key-shifts.
985 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
993 uint8_t read_type = READ_FAIL;
995 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
997 printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
998 for (m = 0; m < 2; m++) {
1000 printf("\n** With bulk-lookup **\n");
1001 read_type |= BULK_LOOKUP;
1003 for (n = 0; n < NUM_TEST; n++) {
1004 unsigned int tot_lcore = rte_lcore_count();
1005 if (tot_lcore < rwc_core_cnt[n] + 1)
1008 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1010 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1011 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
1013 rte_hash_reset(tbl_rwc_test_param.h);
1015 write_type = WRITE_NO_KEY_SHIFT;
1016 if (write_keys(write_type) < 0)
1018 for (i = 1; i <= rwc_core_cnt[n]; i++)
1019 rte_eal_remote_launch(test_rwc_reader,
1020 (void *)(uintptr_t)read_type,
1021 enabled_core_ids[i]);
1022 write_type = WRITE_KEY_SHIFT;
1023 ret = write_keys(write_type);
1028 for (i = 1; i <= rwc_core_cnt[n]; i++)
1029 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1032 unsigned long long cycles_per_lookup =
1033 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
1034 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
1035 rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1036 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1041 rte_hash_free(tbl_rwc_test_param.h);
1045 rte_eal_mp_wait_lcore();
1046 rte_hash_free(tbl_rwc_test_param.h);
1051 * Test lookup perf for multi-writer:
1052 * Reader(s) lookup keys present in the table and likely on the shift-path while
1053 * Writers add keys causing key-shiftsi.
1054 * Writers are running in parallel, on different data plane cores.
1057 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1058 int htm, int ext_bkt)
1060 unsigned int n, m, k;
1064 uint8_t read_type = READ_PASS_SHIFT_PATH;
1066 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1068 printf("\nTest: Multi-add-lookup\n");
1070 for (m = 1; m < NUM_TEST; m++) {
1071 /* Calculate keys added by each writer */
1072 tbl_rwc_test_param.single_insert =
1073 tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1074 for (k = 0; k < 2; k++) {
1076 printf("\n** With bulk-lookup **\n");
1077 read_type |= BULK_LOOKUP;
1079 for (n = 0; n < NUM_TEST; n++) {
1080 unsigned int tot_lcore = rte_lcore_count();
1081 if (tot_lcore < (rwc_core_cnt[n] +
1082 rwc_core_cnt[m] + 1))
1085 printf("\nNumber of writers: %u",
1087 printf("\nNumber of readers: %u\n",
1090 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1091 __atomic_store_n(&gread_cycles, 0,
1094 rte_hash_reset(tbl_rwc_test_param.h);
1096 write_type = WRITE_NO_KEY_SHIFT;
1097 if (write_keys(write_type) < 0)
1100 /* Launch reader(s) */
1101 for (i = 1; i <= rwc_core_cnt[n]; i++)
1102 rte_eal_remote_launch(test_rwc_reader,
1103 (void *)(uintptr_t)read_type,
1104 enabled_core_ids[i]);
1105 write_type = WRITE_KEY_SHIFT;
1108 /* Launch writers */
1109 for (; i <= rwc_core_cnt[m]
1110 + rwc_core_cnt[n]; i++) {
1111 rte_eal_remote_launch
1112 (test_rwc_multi_writer,
1113 (void *)(uintptr_t)pos_core,
1114 enabled_core_ids[i]);
1118 /* Wait for writers to complete */
1119 for (i = rwc_core_cnt[n] + 1;
1120 i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1122 rte_eal_wait_lcore(enabled_core_ids[i]);
1126 for (i = 1; i <= rwc_core_cnt[n]; i++)
1127 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1130 unsigned long long cycles_per_lookup =
1131 __atomic_load_n(&gread_cycles,
1133 __atomic_load_n(&greads,
1135 rwc_perf_results->multi_rw[m][k][n]
1136 = cycles_per_lookup;
1137 printf("Cycles per lookup: %llu\n",
1144 rte_hash_free(tbl_rwc_test_param.h);
1148 rte_eal_mp_wait_lcore();
1149 rte_hash_free(tbl_rwc_test_param.h);
1155 * Reader(s) lookup keys present in the extendable bkt.
1158 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1159 int rwc_lf, int htm, int ext_bkt)
1165 uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1167 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1169 printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1170 for (m = 0; m < 2; m++) {
1172 printf("\n** With bulk-lookup **\n");
1173 read_type |= BULK_LOOKUP;
1175 for (n = 0; n < NUM_TEST; n++) {
1176 unsigned int tot_lcore = rte_lcore_count();
1177 if (tot_lcore < rwc_core_cnt[n] + 1)
1180 printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1182 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1183 __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
1185 rte_hash_reset(tbl_rwc_test_param.h);
1186 write_type = WRITE_NO_KEY_SHIFT;
1187 if (write_keys(write_type) < 0)
1189 write_type = WRITE_KEY_SHIFT;
1190 if (write_keys(write_type) < 0)
1193 for (i = 1; i <= rwc_core_cnt[n]; i++)
1194 rte_eal_remote_launch(test_rwc_reader,
1195 (void *)(uintptr_t)read_type,
1196 enabled_core_ids[i]);
1197 for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1199 if (rte_hash_del_key(tbl_rwc_test_param.h,
1200 tbl_rwc_test_param.keys_ks_extbkt + i)
1202 printf("Delete Failed: %u\n",
1203 tbl_rwc_test_param.keys_ks_extbkt[i]);
1209 for (i = 1; i <= rwc_core_cnt[n]; i++)
1210 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1213 unsigned long long cycles_per_lookup =
1214 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
1215 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
1216 rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1217 = cycles_per_lookup;
1218 printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1223 rte_hash_free(tbl_rwc_test_param.h);
1227 rte_eal_mp_wait_lcore();
1228 rte_hash_free(tbl_rwc_test_param.h);
1232 static struct rte_rcu_qsbr *rv;
1235 * Reader thread using rte_hash data structure with RCU
1238 test_hash_rcu_qsbr_reader(void *arg)
1241 uint32_t num_keys = tbl_rwc_test_param.count_keys_no_ks
1242 - QSBR_REPORTING_INTERVAL;
1243 uint32_t *keys = tbl_rwc_test_param.keys_no_ks;
1244 uint32_t lcore_id = rte_lcore_id();
1247 (void)rte_rcu_qsbr_thread_register(rv, lcore_id);
1248 rte_rcu_qsbr_thread_online(rv, lcore_id);
1250 for (i = 0; i < num_keys; i += j) {
1251 for (j = 0; j < QSBR_REPORTING_INTERVAL; j++)
1252 rte_hash_lookup(tbl_rwc_test_param.h,
1254 /* Update quiescent state counter */
1255 rte_rcu_qsbr_quiescent(rv, lcore_id);
1257 } while (!writer_done);
1258 rte_rcu_qsbr_thread_offline(rv, lcore_id);
1259 (void)rte_rcu_qsbr_thread_unregister(rv, lcore_id);
1265 * Writer thread using rte_hash data structure with RCU
1268 test_hash_rcu_qsbr_writer(void *arg)
1271 uint64_t begin, cycles;
1272 uint8_t pos_core = (uint32_t)((uintptr_t)arg);
1273 offset = pos_core * tbl_rwc_test_param.single_insert;
1275 begin = rte_rdtsc_precise();
1276 for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) {
1277 /* Delete element from the shared data structure */
1278 rte_hash_del_key(tbl_rwc_test_param.h,
1279 tbl_rwc_test_param.keys_no_ks + i);
1280 rte_hash_add_key(tbl_rwc_test_param.h,
1281 tbl_rwc_test_param.keys_no_ks + i);
1283 cycles = rte_rdtsc_precise() - begin;
1284 __atomic_fetch_add(&gwrite_cycles, cycles, __ATOMIC_RELAXED);
1285 __atomic_fetch_add(&gwrites, tbl_rwc_test_param.single_insert,
1291 * Writer perf test with RCU QSBR in DQ mode:
1292 * Writer(s) delete and add keys in the table.
1293 * Readers lookup keys in the hash table
1296 test_hash_rcu_qsbr_writer_perf(struct rwc_perf *rwc_perf_results, int rwc_lf,
1297 int htm, int ext_bkt)
1303 struct rte_hash_rcu_config rcu_config = {0};
1307 printf("\nTest: Writer perf with integrated RCU\n");
1309 if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1312 sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
1313 rv = (struct rte_rcu_qsbr *)rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE);
1316 if (rte_hash_rcu_qsbr_add(tbl_rwc_test_param.h, &rcu_config) < 0) {
1317 printf("RCU init in hash failed\n");
1321 for (n = 0; n < NUM_TEST; n++) {
1322 unsigned int tot_lcore = rte_lcore_count();
1323 if (tot_lcore < rwc_core_cnt[n] + 3)
1326 /* Calculate keys added by each writer */
1327 tbl_rwc_test_param.single_insert =
1328 tbl_rwc_test_param.count_keys_no_ks /
1330 printf("\nNumber of writers: %u\n", rwc_core_cnt[n]);
1332 __atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
1333 __atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
1335 rte_hash_reset(tbl_rwc_test_param.h);
1336 rte_rcu_qsbr_init(rv, RTE_MAX_LCORE);
1338 write_type = WRITE_NO_KEY_SHIFT;
1339 if (write_keys(write_type) < 0)
1341 write_type = WRITE_KEY_SHIFT;
1342 if (write_keys(write_type) < 0)
1345 /* Launch 2 readers */
1346 for (i = 1; i <= 2; i++)
1347 rte_eal_remote_launch(test_hash_rcu_qsbr_reader, NULL,
1348 enabled_core_ids[i]);
1350 /* Launch writer(s) */
1351 for (; i <= rwc_core_cnt[n] + 2; i++) {
1352 rte_eal_remote_launch(test_hash_rcu_qsbr_writer,
1353 (void *)(uintptr_t)pos_core,
1354 enabled_core_ids[i]);
1358 /* Wait for writers to complete */
1359 for (i = 3; i <= rwc_core_cnt[n] + 2; i++)
1360 rte_eal_wait_lcore(enabled_core_ids[i]);
1364 /* Wait for readers to complete */
1365 rte_eal_mp_wait_lcore();
1367 unsigned long long cycles_per_write_operation =
1368 __atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
1369 __atomic_load_n(&gwrites, __ATOMIC_RELAXED);
1370 rwc_perf_results->writer_add_del[n]
1371 = cycles_per_write_operation;
1372 printf("Cycles per write operation: %llu\n",
1373 cycles_per_write_operation);
1377 rte_hash_free(tbl_rwc_test_param.h);
1383 rte_eal_mp_wait_lcore();
1384 rte_hash_free(tbl_rwc_test_param.h);
1390 test_hash_readwrite_lf_perf_main(void)
1393 * Variables used to choose different tests.
1394 * rwc_lf indicates if read-write concurrency lock-free support is
1396 * htm indicates if Hardware transactional memory support is enabled.
1402 if (rte_lcore_count() < 2) {
1403 printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n");
1404 return TEST_SKIPPED;
1407 setlocale(LC_NUMERIC, "");
1409 /* Reset tbl_rwc_test_param to discard values from previous run */
1410 memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param));
1412 if (rte_tm_supported())
1417 if (generate_keys() != 0)
1419 if (get_enabled_cores_list() != 0)
1422 if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1425 printf("Test lookup with read-write concurrency lock free support"
1427 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1430 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1433 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1436 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1439 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1442 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1445 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1448 if (test_hash_rcu_qsbr_writer_perf(&rwc_lf_results, rwc_lf,
1452 printf("\nTest lookup with read-write concurrency lock free support"
1456 printf("With HTM Disabled\n");
1457 if (!RUN_WITH_HTM_DISABLED) {
1458 printf("Enable RUN_WITH_HTM_DISABLED to test with"
1459 " lock-free disabled");
1463 printf("With HTM Enabled\n");
1464 if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1467 if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1470 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1473 if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1476 if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1479 if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1482 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1486 printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1488 for (j = 0; j < 2; j++) {
1490 printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1491 "**********#######\n\n");
1492 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1493 "\t\t\t\t_________________\n");
1494 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1495 "\t\t\tCycles per lookup\n");
1496 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1497 "\t\t\t_________________\n");
1498 for (i = 0; i < NUM_TEST; i++) {
1499 printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1500 printf("Enabled\t\t");
1502 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1503 "%u\n\t\t\t\t\t\t\t\t",
1504 rwc_lf_results.w_no_ks_r_hit[j][i]);
1505 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1506 "%u\n\t\t\t\t\t\t\t\t",
1507 rwc_lf_results.w_no_ks_r_miss[j][i]);
1508 printf("Hash add - key-shifts, lookup - hit"
1509 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1510 rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1511 printf("Hash add - key-shifts, lookup - hit "
1512 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1513 rwc_lf_results.w_ks_r_hit_sp[j][i]);
1514 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1515 "%u\n\t\t\t\t\t\t\t\t",
1516 rwc_lf_results.w_ks_r_miss[j][i]);
1517 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1519 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1521 printf("Disabled\t");
1523 printf("Enabled\t\t");
1525 printf("Disabled\t");
1526 printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1527 "%u\n\t\t\t\t\t\t\t\t",
1528 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1529 printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1530 "%u\n\t\t\t\t\t\t\t\t",
1531 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1532 printf("Hash add - key-shifts, lookup - hit "
1533 "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1534 rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1535 printf("Hash add - key-shifts, lookup - hit "
1536 "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1537 rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1538 printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1539 "%u\n\t\t\t\t\t\t\t\t",
1540 rwc_non_lf_results.w_ks_r_miss[j][i]);
1541 printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1543 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1545 printf("_______\t\t_______\t\t_________\t___\t\t"
1546 "_________\t\t\t\t\t\t_________________\n");
1549 for (i = 1; i < NUM_TEST; i++) {
1550 for (k = 0; k < NUM_TEST; k++) {
1551 printf("%u", rwc_core_cnt[i]);
1552 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1553 printf("Enabled\t\t");
1555 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1557 rwc_lf_results.multi_rw[i][j][k]);
1558 printf("Disabled\t");
1560 printf("Enabled\t\t");
1562 printf("Disabled\t");
1563 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1564 rwc_non_lf_results.multi_rw[i][j][k]);
1566 printf("_______\t\t_______\t\t_________\t___"
1567 "\t\t_________\t\t\t\t\t\t"
1568 "_________________\n");
1572 rte_free(tbl_rwc_test_param.keys);
1573 rte_free(tbl_rwc_test_param.keys_no_ks);
1574 rte_free(tbl_rwc_test_param.keys_ks);
1575 rte_free(tbl_rwc_test_param.keys_absent);
1576 rte_free(tbl_rwc_test_param.keys_shift_path);
1577 rte_free(tbl_rwc_test_param.keys_non_shift_path);
1578 rte_free(tbl_rwc_test_param.keys_ext_bkt);
1579 rte_free(tbl_rwc_test_param.keys_ks_extbkt);
1583 REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest,
1584 test_hash_readwrite_lf_perf_main);