From: Honnappa Nagarahalli Date: Mon, 3 Feb 2020 19:49:11 +0000 (-0600) Subject: test/hash: move lock-free tests to perf tests X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=2cc1d7b40f40768b0aca71450f4b011eff27c919;p=dpdk.git test/hash: move lock-free tests to perf tests Move reader writer lock free tests to performance tests. Signed-off-by: Honnappa Nagarahalli --- diff --git a/app/test/Makefile b/app/test/Makefile index 57930c00b1..d955dbb03f 100644 --- a/app/test/Makefile +++ b/app/test/Makefile @@ -122,7 +122,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_multiwriter.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite.c -SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite_lf.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite_lf_perf.c SRCS-$(CONFIG_RTE_LIBRTE_RIB) += test_rib.c SRCS-$(CONFIG_RTE_LIBRTE_RIB) += test_rib6.c diff --git a/app/test/autotest_data.py b/app/test/autotest_data.py index 71db4b3f67..7b1d01389b 100644 --- a/app/test/autotest_data.py +++ b/app/test/autotest_data.py @@ -676,8 +676,8 @@ non_parallel_test_list = [ "Report": None, }, { - "Name": "Hash read-write lock-free concurrency autotest", - "Command": "hash_readwrite_lf_autotest", + "Name": "Hash read-write lock-free concurrency perf autotest", + "Command": "hash_readwrite_lf_perf_autotest", "Func": default_autotest, "Report": None, }, diff --git a/app/test/meson.build b/app/test/meson.build index 08c0ecb3fb..3675ffb5c5 100644 --- a/app/test/meson.build +++ b/app/test/meson.build @@ -59,7 +59,7 @@ test_sources = files('commands.c', 'test_hash_multiwriter.c', 'test_hash_readwrite.c', 'test_hash_perf.c', - 'test_hash_readwrite_lf.c', + 'test_hash_readwrite_lf_perf.c', 'test_interrupts.c', 'test_ipsec.c', 'test_ipsec_sad.c', @@ -234,7 +234,6 @@ fast_test_names = [ 'eventdev_common_autotest', 'fbarray_autotest', 'hash_readwrite_func_autotest', - 'hash_readwrite_lf_autotest', 'ipsec_autotest', 'kni_autotest', 'kvargs_autotest', @@ -283,6 +282,7 @@ perf_test_names = [ 'stack_lf_perf_autotest', 'rand_perf_autotest', 'hash_readwrite_perf_autotest', + 'hash_readwrite_lf_perf_autotest', ] driver_test_names = [ diff --git a/app/test/test_hash_readwrite_lf.c b/app/test/test_hash_readwrite_lf.c deleted file mode 100644 index 97c304054c..0000000000 --- a/app/test/test_hash_readwrite_lf.c +++ /dev/null @@ -1,1434 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2018 Arm Limited - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "test.h" - -#ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF -#define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0 -#endif - -#define BULK_LOOKUP_SIZE 32 - -#define RUN_WITH_HTM_DISABLED 0 - -#if (RUN_WITH_HTM_DISABLED) - -#define TOTAL_ENTRY (5*1024) -#define TOTAL_INSERT (5*1024) - -#else - -#define TOTAL_ENTRY (4*1024*1024) -#define TOTAL_INSERT (4*1024*1024) - -#endif - -#define READ_FAIL 1 -#define READ_PASS_NO_KEY_SHIFTS 2 -#define READ_PASS_SHIFT_PATH 4 -#define READ_PASS_NON_SHIFT_PATH 8 -#define BULK_LOOKUP 16 -#define READ_PASS_KEY_SHIFTS_EXTBKT 32 - -#define WRITE_NO_KEY_SHIFT 0 -#define WRITE_KEY_SHIFT 1 -#define WRITE_EXT_BKT 2 - -#define NUM_TEST 3 -static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4}; - -struct rwc_perf { - uint32_t w_no_ks_r_hit[2][NUM_TEST]; - uint32_t w_no_ks_r_miss[2][NUM_TEST]; - uint32_t w_ks_r_hit_nsp[2][NUM_TEST]; - uint32_t w_ks_r_hit_sp[2][NUM_TEST]; - uint32_t w_ks_r_miss[2][NUM_TEST]; - uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST]; - uint32_t w_ks_r_hit_extbkt[2][NUM_TEST]; -}; - -static struct rwc_perf rwc_lf_results, rwc_non_lf_results; - -static struct { - uint32_t *keys; - uint32_t *keys_no_ks; - uint32_t *keys_ks; - uint32_t *keys_absent; - uint32_t *keys_shift_path; - uint32_t *keys_non_shift_path; - uint32_t *keys_ext_bkt; - uint32_t *keys_ks_extbkt; - uint32_t count_keys_no_ks; - uint32_t count_keys_ks; - uint32_t count_keys_absent; - uint32_t count_keys_shift_path; - uint32_t count_keys_non_shift_path; - uint32_t count_keys_extbkt; - uint32_t count_keys_ks_extbkt; - uint32_t single_insert; - struct rte_hash *h; -} tbl_rwc_test_param; - -static rte_atomic64_t gread_cycles; -static rte_atomic64_t greads; - -static volatile uint8_t writer_done; - -static uint16_t enabled_core_ids[RTE_MAX_LCORE]; - -static uint8_t *scanned_bkts; - -static inline uint16_t -get_short_sig(const hash_sig_t hash) -{ - return hash >> 16; -} - -static inline uint32_t -get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h, - const hash_sig_t hash) -{ - uint32_t num_buckets; - uint32_t bucket_bitmask; - num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; - bucket_bitmask = num_buckets - 1; - return hash & bucket_bitmask; -} - -static inline uint32_t -get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h, - uint32_t cur_bkt_idx, uint16_t sig) -{ - uint32_t num_buckets; - uint32_t bucket_bitmask; - num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; - bucket_bitmask = num_buckets - 1; - return (cur_bkt_idx ^ sig) & bucket_bitmask; -} - - -static inline int -get_enabled_cores_list(void) -{ - uint32_t i = 0; - uint16_t core_id; - uint32_t max_cores = rte_lcore_count(); - RTE_LCORE_FOREACH(core_id) { - enabled_core_ids[i] = core_id; - i++; - } - - if (i != max_cores) { - printf("Number of enabled cores in list is different from " - "number given by rte_lcore_count()\n"); - return -1; - } - return 0; -} - -static int -init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt) -{ - struct rte_hash *handle; - - struct rte_hash_parameters hash_params = { - .entries = TOTAL_ENTRY, - .key_len = sizeof(uint32_t), - .hash_func_init_val = 0, - .socket_id = rte_socket_id(), - }; - - if (use_jhash) - hash_params.hash_func = rte_jhash; - else - hash_params.hash_func = rte_hash_crc; - - if (rwc_lf) - hash_params.extra_flag = - RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF | - RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; - else if (htm) - hash_params.extra_flag = - RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | - RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | - RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; - else - hash_params.extra_flag = - RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | - RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; - - if (ext_bkt) - hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE; - - hash_params.name = "tests"; - - handle = rte_hash_create(&hash_params); - if (handle == NULL) { - printf("hash creation failed"); - return -1; - } - - tbl_rwc_test_param.h = handle; - return 0; -} - -static inline int -check_bucket(uint32_t bkt_idx, uint32_t key) -{ - uint32_t iter; - uint32_t prev_iter; - uint32_t diff; - uint32_t count = 0; - const void *next_key; - void *next_data; - - /* Temporary bucket to hold the keys */ - uint32_t keys_in_bkt[8]; - - iter = bkt_idx * 8; - prev_iter = iter; - while (rte_hash_iterate(tbl_rwc_test_param.h, - &next_key, &next_data, &iter) >= 0) { - - /* Check for duplicate entries */ - if (*(const uint32_t *)next_key == key) - return 1; - - /* Identify if there is any free entry in the bucket */ - diff = iter - prev_iter; - if (diff > 1) - break; - - prev_iter = iter; - keys_in_bkt[count] = *(const uint32_t *)next_key; - count++; - - /* All entries in the bucket are occupied */ - if (count == 8) { - - /* - * Check if bucket was not scanned before, to avoid - * duplicate keys. - */ - if (scanned_bkts[bkt_idx] == 0) { - /* - * Since this bucket (pointed to by bkt_idx) is - * full, it is likely that key(s) in this - * bucket will be on the shift path, when - * collision occurs. Thus, add it to - * keys_shift_path. - */ - memcpy(tbl_rwc_test_param.keys_shift_path + - tbl_rwc_test_param.count_keys_shift_path - , keys_in_bkt, 32); - tbl_rwc_test_param.count_keys_shift_path += 8; - scanned_bkts[bkt_idx] = 1; - } - return -1; - } - } - return 0; -} - -static int -generate_keys(void) -{ - uint32_t *keys = NULL; - uint32_t *keys_no_ks = NULL; - uint32_t *keys_ks = NULL; - uint32_t *keys_absent = NULL; - uint32_t *keys_non_shift_path = NULL; - uint32_t *keys_ext_bkt = NULL; - uint32_t *keys_ks_extbkt = NULL; - uint32_t *found = NULL; - uint32_t count_keys_no_ks = 0; - uint32_t count_keys_ks = 0; - uint32_t count_keys_extbkt = 0; - uint32_t i; - - if (init_params(0, 0, 0, 0) != 0) - return -1; - - /* - * keys will consist of a) keys whose addition to the hash table - * will result in shifting of the existing keys to their alternate - * locations b) keys whose addition to the hash table will not result - * in shifting of the existing keys. - */ - keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (keys == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that - * will NOT result in shifting of the existing keys to their alternate - * locations. Roughly around 900K keys. - */ - keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (keys_no_ks == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will - * result in shifting of the existing keys to their alternate locations. - * Roughly around 146K keys. There might be repeating keys. More code is - * required to filter out these keys which will complicate the test case - */ - keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (keys_ks == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* Used to identify keys not inserted in the hash table */ - found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (found == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * This consist of keys not inserted to the hash table. - * Used to test perf of lookup on keys that do not exist in the table. - */ - keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (keys_absent == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * This consist of keys which are likely to be on the shift - * path (i.e. being moved to alternate location), when collision occurs - * on addition of a key to an already full primary bucket. - * Used to test perf of lookup on keys that are on the shift path. - */ - tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) * - TOTAL_INSERT, 0); - if (tbl_rwc_test_param.keys_shift_path == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * This consist of keys which are never on the shift - * path (i.e. being moved to alternate location), when collision occurs - * on addition of a key to an already full primary bucket. - * Used to test perf of lookup on keys that are not on the shift path. - */ - keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, - 0); - if (keys_non_shift_path == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * This consist of keys which will be stored in extended buckets - */ - keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (keys_ext_bkt == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - /* - * This consist of keys which when deleted causes shifting of keys - * in extended buckets to respective secondary buckets - */ - keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); - if (keys_ks_extbkt == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - hash_sig_t sig; - uint32_t prim_bucket_idx; - uint32_t sec_bucket_idx; - uint16_t short_sig; - uint32_t num_buckets; - num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; - int ret; - - /* - * Used to mark bkts in which at least one key was shifted to its - * alternate location - */ - scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0); - if (scanned_bkts == NULL) { - printf("RTE_MALLOC failed\n"); - goto err; - } - - tbl_rwc_test_param.keys = keys; - tbl_rwc_test_param.keys_no_ks = keys_no_ks; - tbl_rwc_test_param.keys_ks = keys_ks; - tbl_rwc_test_param.keys_absent = keys_absent; - tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path; - tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt; - tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt; - /* Generate keys by adding previous two keys, neglect overflow */ - printf("Generating keys...\n"); - keys[0] = 0; - keys[1] = 1; - for (i = 2; i < TOTAL_INSERT; i++) - keys[i] = keys[i-1] + keys[i-2]; - - /* Segregate keys into keys_no_ks and keys_ks */ - for (i = 0; i < TOTAL_INSERT; i++) { - /* Check if primary bucket has space.*/ - sig = rte_hash_hash(tbl_rwc_test_param.h, - tbl_rwc_test_param.keys+i); - prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h, - sig); - ret = check_bucket(prim_bucket_idx, keys[i]); - if (ret < 0) { - /* - * Primary bucket is full, this key will result in - * shifting of the keys to their alternate locations. - */ - keys_ks[count_keys_ks] = keys[i]; - count_keys_ks++; - } else if (ret == 0) { - /* - * Primary bucket has space, this key will not result in - * shifting of the keys. Hence, add key to the table. - */ - ret = rte_hash_add_key_data(tbl_rwc_test_param.h, - keys+i, - (void *)((uintptr_t)i)); - if (ret < 0) { - printf("writer failed %"PRIu32"\n", i); - break; - } - keys_no_ks[count_keys_no_ks] = keys[i]; - count_keys_no_ks++; - } - } - - for (i = 0; i < count_keys_no_ks; i++) { - /* - * Identify keys in keys_no_ks with value less than - * 4M (HTM enabled) OR 5K (HTM disabled) - */ - if (keys_no_ks[i] < TOTAL_INSERT) - found[keys_no_ks[i]]++; - } - - for (i = 0; i < count_keys_ks; i++) { - /* - * Identify keys in keys_ks with value less than - * 4M (HTM enabled) OR 5K (HTM disabled) - */ - if (keys_ks[i] < TOTAL_INSERT) - found[keys_ks[i]]++; - } - - uint32_t count_keys_absent = 0; - for (i = 0; i < TOTAL_INSERT; i++) { - /* - * Identify missing keys between 0 and - * 4M (HTM enabled) OR 5K (HTM disabled) - */ - if (found[i] == 0) - keys_absent[count_keys_absent++] = i; - } - - /* Find keys that will not be on the shift path */ - uint32_t iter; - const void *next_key; - void *next_data; - uint32_t count = 0; - for (i = 0; i < num_buckets; i++) { - /* Check bucket for no keys shifted to alternate locations */ - if (scanned_bkts[i] == 0) { - iter = i * 8; - while (rte_hash_iterate(tbl_rwc_test_param.h, - &next_key, &next_data, &iter) >= 0) { - - /* Check if key belongs to the current bucket */ - if (i >= (iter-1)/8) - keys_non_shift_path[count++] - = *(const uint32_t *)next_key; - else - break; - } - } - } - - tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks; - tbl_rwc_test_param.count_keys_ks = count_keys_ks; - tbl_rwc_test_param.count_keys_absent = count_keys_absent; - tbl_rwc_test_param.count_keys_non_shift_path = count; - - memset(scanned_bkts, 0, num_buckets); - count = 0; - /* Find keys that will be in extended buckets */ - for (i = 0; i < count_keys_ks; i++) { - ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i); - if (ret < 0) { - /* Key will be added to ext bkt */ - keys_ext_bkt[count_keys_extbkt++] = keys_ks[i]; - /* Sec bkt to be added to keys_ks_extbkt */ - sig = rte_hash_hash(tbl_rwc_test_param.h, - tbl_rwc_test_param.keys_ks + i); - prim_bucket_idx = get_prim_bucket_index( - tbl_rwc_test_param.h, sig); - short_sig = get_short_sig(sig); - sec_bucket_idx = get_alt_bucket_index( - tbl_rwc_test_param.h, - prim_bucket_idx, short_sig); - if (scanned_bkts[sec_bucket_idx] == 0) - scanned_bkts[sec_bucket_idx] = 1; - } - } - - /* Find keys that will shift keys in ext bucket*/ - for (i = 0; i < num_buckets; i++) { - if (scanned_bkts[i] == 1) { - iter = i * 8; - while (rte_hash_iterate(tbl_rwc_test_param.h, - &next_key, &next_data, &iter) >= 0) { - /* Check if key belongs to the current bucket */ - if (i >= (iter-1)/8) - keys_ks_extbkt[count++] - = *(const uint32_t *)next_key; - else - break; - } - } - } - - tbl_rwc_test_param.count_keys_ks_extbkt = count; - tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt; - - printf("\nCount of keys NOT causing shifting of existing keys to " - "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks); - printf("\nCount of keys causing shifting of existing keys to alternate " - "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks); - printf("Count of absent keys that will never be added to the hash " - "table: %d\n\n", tbl_rwc_test_param.count_keys_absent); - printf("Count of keys likely to be on the shift path: %d\n\n", - tbl_rwc_test_param.count_keys_shift_path); - printf("Count of keys not likely to be on the shift path: %d\n\n", - tbl_rwc_test_param.count_keys_non_shift_path); - printf("Count of keys in extended buckets: %d\n\n", - tbl_rwc_test_param.count_keys_extbkt); - printf("Count of keys shifting keys in ext buckets: %d\n\n", - tbl_rwc_test_param.count_keys_ks_extbkt); - - rte_free(found); - rte_free(scanned_bkts); - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_free(keys); - rte_free(keys_no_ks); - rte_free(keys_ks); - rte_free(keys_absent); - rte_free(found); - rte_free(tbl_rwc_test_param.keys_shift_path); - rte_free(keys_non_shift_path); - rte_free(keys_ext_bkt); - rte_free(keys_ks_extbkt); - rte_free(scanned_bkts); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -static int -test_rwc_reader(__attribute__((unused)) void *arg) -{ - uint32_t i, j; - int ret; - uint64_t begin, cycles; - uint32_t loop_cnt = 0; - uint8_t read_type = (uint8_t)((uintptr_t)arg); - uint32_t read_cnt; - uint32_t *keys; - uint32_t extra_keys; - int32_t pos[BULK_LOOKUP_SIZE]; - void *temp_a[BULK_LOOKUP_SIZE]; - - if (read_type & READ_FAIL) { - keys = tbl_rwc_test_param.keys_absent; - read_cnt = tbl_rwc_test_param.count_keys_absent; - } else if (read_type & READ_PASS_NO_KEY_SHIFTS) { - keys = tbl_rwc_test_param.keys_no_ks; - read_cnt = tbl_rwc_test_param.count_keys_no_ks; - } else if (read_type & READ_PASS_SHIFT_PATH) { - keys = tbl_rwc_test_param.keys_shift_path; - read_cnt = tbl_rwc_test_param.count_keys_shift_path; - } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) { - keys = tbl_rwc_test_param.keys_ext_bkt; - read_cnt = tbl_rwc_test_param.count_keys_extbkt; - } else { - keys = tbl_rwc_test_param.keys_non_shift_path; - read_cnt = tbl_rwc_test_param.count_keys_non_shift_path; - } - - extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1); - - begin = rte_rdtsc_precise(); - do { - if (read_type & BULK_LOOKUP) { - for (i = 0; i < (read_cnt - extra_keys); - i += BULK_LOOKUP_SIZE) { - /* Array of pointer to the list of keys */ - for (j = 0; j < BULK_LOOKUP_SIZE; j++) - temp_a[j] = keys + i + j; - rte_hash_lookup_bulk(tbl_rwc_test_param.h, - (const void **) - ((uintptr_t)temp_a), - BULK_LOOKUP_SIZE, pos); - /* Validate lookup result */ - for (j = 0; j < BULK_LOOKUP_SIZE; j++) - if ((read_type & READ_FAIL && - pos[j] != -ENOENT) || - (!(read_type & READ_FAIL) && - pos[j] == -ENOENT)) { - printf("lookup failed!" - "%"PRIu32"\n", - keys[i + j]); - return -1; - } - } - for (j = 0; j < extra_keys; j++) - temp_a[j] = keys + i + j; - - rte_hash_lookup_bulk(tbl_rwc_test_param.h, - (const void **) - ((uintptr_t)temp_a), - extra_keys, pos); - for (j = 0; j < extra_keys; j++) - if ((read_type & READ_FAIL && - pos[j] != -ENOENT) || - (!(read_type & READ_FAIL) && - pos[j] == -ENOENT)) { - printf("lookup failed! %"PRIu32"\n", - keys[i + j]); - return -1; - } - } else { - for (i = 0; i < read_cnt; i++) { - ret = rte_hash_lookup - (tbl_rwc_test_param.h, keys + i); - if (((read_type & READ_FAIL) && - (ret != -ENOENT)) || - (!(read_type & READ_FAIL) && - ret == -ENOENT)) { - printf("lookup failed! %"PRIu32"\n", - keys[i]); - return -1; - } - } - } - loop_cnt++; - } while (!writer_done); - - cycles = rte_rdtsc_precise() - begin; - rte_atomic64_add(&gread_cycles, cycles); - rte_atomic64_add(&greads, read_cnt*loop_cnt); - return 0; -} - -static int -write_keys(uint8_t write_type) -{ - uint32_t i; - int ret; - uint32_t key_cnt = 0; - uint32_t *keys; - if (write_type == WRITE_KEY_SHIFT) { - key_cnt = tbl_rwc_test_param.count_keys_ks; - keys = tbl_rwc_test_param.keys_ks; - } else if (write_type == WRITE_NO_KEY_SHIFT) { - key_cnt = tbl_rwc_test_param.count_keys_no_ks; - keys = tbl_rwc_test_param.keys_no_ks; - } else if (write_type == WRITE_EXT_BKT) { - key_cnt = tbl_rwc_test_param.count_keys_extbkt; - keys = tbl_rwc_test_param.keys_ext_bkt; - } - for (i = 0; i < key_cnt; i++) { - ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i); - if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) { - printf("writer failed %"PRIu32"\n", i); - return -1; - } - } - return 0; -} - -static int -test_rwc_multi_writer(__attribute__((unused)) void *arg) -{ - uint32_t i, offset; - uint32_t pos_core = (uint32_t)((uintptr_t)arg); - offset = pos_core * tbl_rwc_test_param.single_insert; - for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) - rte_hash_add_key(tbl_rwc_test_param.h, - tbl_rwc_test_param.keys_ks + i); - return 0; -} - -/* - * Test lookup perf: - * Reader(s) lookup keys present in the table. - */ -static int -test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf, - int htm, int ext_bkt) -{ - unsigned int n, m; - uint64_t i; - int use_jhash = 0; - uint8_t write_type = WRITE_NO_KEY_SHIFT; - uint8_t read_type = READ_PASS_NO_KEY_SHIFTS; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Hash add - no key-shifts, read - hit\n"); - for (m = 0; m < 2; m++) { - if (m == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < rwc_core_cnt[n] + 1) - goto finish; - - printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); - - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - writer_done = 0; - if (write_keys(write_type) < 0) - goto err; - writer_done = 1; - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) / - rte_atomic64_read(&greads); - rwc_perf_results->w_no_ks_r_hit[m][n] - = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", cycles_per_lookup); - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -/* - * Test lookup perf: - * Reader(s) lookup keys absent in the table while - * 'Main' thread adds with no key-shifts. - */ -static int -test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, - int htm, int ext_bkt) -{ - unsigned int n, m; - uint64_t i; - int use_jhash = 0; - uint8_t write_type = WRITE_NO_KEY_SHIFT; - uint8_t read_type = READ_FAIL; - int ret; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n"); - for (m = 0; m < 2; m++) { - if (m == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < rwc_core_cnt[n] + 1) - goto finish; - - printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); - - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - writer_done = 0; - - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - ret = write_keys(write_type); - writer_done = 1; - - if (ret < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) / - rte_atomic64_read(&greads); - rwc_perf_results->w_no_ks_r_miss[m][n] - = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", cycles_per_lookup); - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -/* - * Test lookup perf: - * Reader(s) lookup keys present in the table and not likely to be on the - * shift path while 'Main' thread adds keys causing key-shifts. - */ -static int -test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results, - int rwc_lf, int htm, int ext_bkt) -{ - unsigned int n, m; - uint64_t i; - int use_jhash = 0; - int ret; - uint8_t write_type; - uint8_t read_type = READ_PASS_NON_SHIFT_PATH; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Hash add - key shift, Hash lookup - hit" - " (non-shift-path)\n"); - for (m = 0; m < 2; m++) { - if (m == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < rwc_core_cnt[n] + 1) - goto finish; - - printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); - - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - writer_done = 0; - write_type = WRITE_NO_KEY_SHIFT; - if (write_keys(write_type) < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - write_type = WRITE_KEY_SHIFT; - ret = write_keys(write_type); - writer_done = 1; - - if (ret < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) / - rte_atomic64_read(&greads); - rwc_perf_results->w_ks_r_hit_nsp[m][n] - = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", cycles_per_lookup); - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -/* - * Test lookup perf: - * Reader(s) lookup keys present in the table and likely on the shift-path while - * 'Main' thread adds keys causing key-shifts. - */ -static int -test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf, - int htm, int ext_bkt) -{ - unsigned int n, m; - uint64_t i; - int use_jhash = 0; - int ret; - uint8_t write_type; - uint8_t read_type = READ_PASS_SHIFT_PATH; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)" - "\n"); - - for (m = 0; m < 2; m++) { - if (m == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < rwc_core_cnt[n] + 1) - goto finish; - - printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - writer_done = 0; - write_type = WRITE_NO_KEY_SHIFT; - if (write_keys(write_type) < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - write_type = WRITE_KEY_SHIFT; - ret = write_keys(write_type); - writer_done = 1; - - if (ret < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) / - rte_atomic64_read(&greads); - rwc_perf_results->w_ks_r_hit_sp[m][n] - = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", cycles_per_lookup); - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -/* - * Test lookup perf: - * Reader(s) lookup keys absent in the table while - * 'Main' thread adds keys causing key-shifts. - */ -static int -test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int - htm, int ext_bkt) -{ - unsigned int n, m; - uint64_t i; - int use_jhash = 0; - int ret; - uint8_t write_type; - uint8_t read_type = READ_FAIL; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Hash add - key shift, Hash lookup - miss\n"); - for (m = 0; m < 2; m++) { - if (m == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < rwc_core_cnt[n] + 1) - goto finish; - - printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); - - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - writer_done = 0; - write_type = WRITE_NO_KEY_SHIFT; - if (write_keys(write_type) < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - write_type = WRITE_KEY_SHIFT; - ret = write_keys(write_type); - writer_done = 1; - - if (ret < 0) - goto err; - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) / - rte_atomic64_read(&greads); - rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", cycles_per_lookup); - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -/* - * Test lookup perf for multi-writer: - * Reader(s) lookup keys present in the table and likely on the shift-path while - * Writers add keys causing key-shiftsi. - * Writers are running in parallel, on different data plane cores. - */ -static int -test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf, - int htm, int ext_bkt) -{ - unsigned int n, m, k; - uint64_t i; - int use_jhash = 0; - uint8_t write_type; - uint8_t read_type = READ_PASS_SHIFT_PATH; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Multi-add-lookup\n"); - uint8_t pos_core; - for (m = 1; m < NUM_TEST; m++) { - /* Calculate keys added by each writer */ - tbl_rwc_test_param.single_insert = - tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m]; - for (k = 0; k < 2; k++) { - if (k == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < (rwc_core_cnt[n] + - rwc_core_cnt[m] + 1)) - goto finish; - - printf("\nNumber of writers: %u", - rwc_core_cnt[m]); - printf("\nNumber of readers: %u\n", - rwc_core_cnt[n]); - - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - writer_done = 0; - write_type = WRITE_NO_KEY_SHIFT; - if (write_keys(write_type) < 0) - goto err; - - /* Launch reader(s) */ - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - write_type = WRITE_KEY_SHIFT; - pos_core = 0; - - /* Launch writers */ - for (; i <= rwc_core_cnt[m] - + rwc_core_cnt[n]; i++) { - rte_eal_remote_launch - (test_rwc_multi_writer, - (void *)(uintptr_t)pos_core, - enabled_core_ids[i]); - pos_core++; - } - - /* Wait for writers to complete */ - for (i = rwc_core_cnt[n] + 1; - i <= rwc_core_cnt[m] + rwc_core_cnt[n]; - i++) - rte_eal_wait_lcore(enabled_core_ids[i]); - - writer_done = 1; - - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) - / rte_atomic64_read(&greads); - rwc_perf_results->multi_rw[m][k][n] - = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", - cycles_per_lookup); - } - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -/* - * Test lookup perf: - * Reader(s) lookup keys present in the extendable bkt. - */ -static int -test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results, - int rwc_lf, int htm, int ext_bkt) -{ - unsigned int n, m; - uint64_t i; - int use_jhash = 0; - uint8_t write_type; - uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT; - - rte_atomic64_init(&greads); - rte_atomic64_init(&gread_cycles); - - if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) - goto err; - printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n"); - for (m = 0; m < 2; m++) { - if (m == 1) { - printf("\n** With bulk-lookup **\n"); - read_type |= BULK_LOOKUP; - } - for (n = 0; n < NUM_TEST; n++) { - unsigned int tot_lcore = rte_lcore_count(); - if (tot_lcore < rwc_core_cnt[n] + 1) - goto finish; - - printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); - - rte_atomic64_clear(&greads); - rte_atomic64_clear(&gread_cycles); - - rte_hash_reset(tbl_rwc_test_param.h); - write_type = WRITE_NO_KEY_SHIFT; - if (write_keys(write_type) < 0) - goto err; - write_type = WRITE_KEY_SHIFT; - if (write_keys(write_type) < 0) - goto err; - writer_done = 0; - for (i = 1; i <= rwc_core_cnt[n]; i++) - rte_eal_remote_launch(test_rwc_reader, - (void *)(uintptr_t)read_type, - enabled_core_ids[i]); - for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt; - i++) { - if (rte_hash_del_key(tbl_rwc_test_param.h, - tbl_rwc_test_param.keys_ks_extbkt + i) - < 0) { - printf("Delete Failed: %u\n", - tbl_rwc_test_param.keys_ks_extbkt[i]); - goto err; - } - } - writer_done = 1; - - for (i = 1; i <= rwc_core_cnt[n]; i++) - if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) - goto err; - - unsigned long long cycles_per_lookup = - rte_atomic64_read(&gread_cycles) / - rte_atomic64_read(&greads); - rwc_perf_results->w_ks_r_hit_extbkt[m][n] - = cycles_per_lookup; - printf("Cycles per lookup: %llu\n", cycles_per_lookup); - } - } - -finish: - rte_hash_free(tbl_rwc_test_param.h); - return 0; - -err: - rte_eal_mp_wait_lcore(); - rte_hash_free(tbl_rwc_test_param.h); - return -1; -} - -static int -test_hash_readwrite_lf_main(void) -{ - /* - * Variables used to choose different tests. - * rwc_lf indicates if read-write concurrency lock-free support is - * enabled. - * htm indicates if Hardware transactional memory support is enabled. - */ - int rwc_lf = 0; - int htm; - int ext_bkt = 0; - - if (rte_lcore_count() < 2) { - printf("Not enough cores for hash_readwrite_lf_autotest, expecting at least 2\n"); - return TEST_SKIPPED; - } - - setlocale(LC_NUMERIC, ""); - - /* Reset tbl_rwc_test_param to discard values from previous run */ - memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param)); - - if (rte_tm_supported()) - htm = 1; - else - htm = 0; - - if (generate_keys() != 0) - return -1; - if (get_enabled_cores_list() != 0) - return -1; - - if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) { - rwc_lf = 1; - ext_bkt = 1; - printf("Test lookup with read-write concurrency lock free support" - " enabled\n"); - if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; - if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; - } - printf("\nTest lookup with read-write concurrency lock free support" - " disabled\n"); - rwc_lf = 0; - if (!htm) { - printf("With HTM Disabled\n"); - if (!RUN_WITH_HTM_DISABLED) { - printf("Enable RUN_WITH_HTM_DISABLED to test with" - " lock-free disabled"); - goto results; - } - } else - printf("With HTM Enabled\n"); - if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm, - ext_bkt) < 0) - return -1; - if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf, - htm, ext_bkt) < 0) - return -1; -results: - printf("\n\t\t\t\t\t\t********** Results summary **********\n\n"); - int i, j, k; - for (j = 0; j < 2; j++) { - if (j == 1) - printf("\n\t\t\t\t\t#######********** Bulk Lookup " - "**********#######\n\n"); - printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t" - "\t\t\t\t_________________\n"); - printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t" - "\t\t\tCycles per lookup\n"); - printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t" - "\t\t\t_________________\n"); - for (i = 0; i < NUM_TEST; i++) { - printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]); - printf("Enabled\t\t"); - printf("N/A\t\t"); - printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" - "%u\n\t\t\t\t\t\t\t\t", - rwc_lf_results.w_no_ks_r_hit[j][i]); - printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" - "%u\n\t\t\t\t\t\t\t\t", - rwc_lf_results.w_no_ks_r_miss[j][i]); - printf("Hash add - key-shifts, lookup - hit" - "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", - rwc_lf_results.w_ks_r_hit_nsp[j][i]); - printf("Hash add - key-shifts, lookup - hit " - "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", - rwc_lf_results.w_ks_r_hit_sp[j][i]); - printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" - "%u\n\t\t\t\t\t\t\t\t", - rwc_lf_results.w_ks_r_miss[j][i]); - printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" - "%u\n\n\t\t\t\t", - rwc_lf_results.w_ks_r_hit_extbkt[j][i]); - - printf("Disabled\t"); - if (htm) - printf("Enabled\t\t"); - else - printf("Disabled\t"); - printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" - "%u\n\t\t\t\t\t\t\t\t", - rwc_non_lf_results.w_no_ks_r_hit[j][i]); - printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" - "%u\n\t\t\t\t\t\t\t\t", - rwc_non_lf_results.w_no_ks_r_miss[j][i]); - printf("Hash add - key-shifts, lookup - hit " - "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", - rwc_non_lf_results.w_ks_r_hit_nsp[j][i]); - printf("Hash add - key-shifts, lookup - hit " - "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", - rwc_non_lf_results.w_ks_r_hit_sp[j][i]); - printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" - "%u\n\t\t\t\t\t\t\t\t", - rwc_non_lf_results.w_ks_r_miss[j][i]); - printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" - "%u\n", - rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]); - - printf("_______\t\t_______\t\t_________\t___\t\t" - "_________\t\t\t\t\t\t_________________\n"); - } - - for (i = 1; i < NUM_TEST; i++) { - for (k = 0; k < NUM_TEST; k++) { - printf("%u", rwc_core_cnt[i]); - printf("\t\t%u\t\t", rwc_core_cnt[k]); - printf("Enabled\t\t"); - printf("N/A\t\t"); - printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t" - "\t\t", - rwc_lf_results.multi_rw[i][j][k]); - printf("Disabled\t"); - if (htm) - printf("Enabled\t\t"); - else - printf("Disabled\t"); - printf("Multi-add-lookup\t\t\t\t\t\t%u\n", - rwc_non_lf_results.multi_rw[i][j][k]); - - printf("_______\t\t_______\t\t_________\t___" - "\t\t_________\t\t\t\t\t\t" - "_________________\n"); - } - } - } - rte_free(tbl_rwc_test_param.keys); - rte_free(tbl_rwc_test_param.keys_no_ks); - rte_free(tbl_rwc_test_param.keys_ks); - rte_free(tbl_rwc_test_param.keys_absent); - rte_free(tbl_rwc_test_param.keys_shift_path); - rte_free(tbl_rwc_test_param.keys_non_shift_path); - rte_free(tbl_rwc_test_param.keys_ext_bkt); - rte_free(tbl_rwc_test_param.keys_ks_extbkt); - return 0; -} - -REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main); diff --git a/app/test/test_hash_readwrite_lf_perf.c b/app/test/test_hash_readwrite_lf_perf.c new file mode 100644 index 0000000000..7bfc067f4e --- /dev/null +++ b/app/test/test_hash_readwrite_lf_perf.c @@ -0,0 +1,1435 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Arm Limited + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test.h" + +#ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF +#define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0 +#endif + +#define BULK_LOOKUP_SIZE 32 + +#define RUN_WITH_HTM_DISABLED 0 + +#if (RUN_WITH_HTM_DISABLED) + +#define TOTAL_ENTRY (5*1024) +#define TOTAL_INSERT (5*1024) + +#else + +#define TOTAL_ENTRY (4*1024*1024) +#define TOTAL_INSERT (4*1024*1024) + +#endif + +#define READ_FAIL 1 +#define READ_PASS_NO_KEY_SHIFTS 2 +#define READ_PASS_SHIFT_PATH 4 +#define READ_PASS_NON_SHIFT_PATH 8 +#define BULK_LOOKUP 16 +#define READ_PASS_KEY_SHIFTS_EXTBKT 32 + +#define WRITE_NO_KEY_SHIFT 0 +#define WRITE_KEY_SHIFT 1 +#define WRITE_EXT_BKT 2 + +#define NUM_TEST 3 +static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4}; + +struct rwc_perf { + uint32_t w_no_ks_r_hit[2][NUM_TEST]; + uint32_t w_no_ks_r_miss[2][NUM_TEST]; + uint32_t w_ks_r_hit_nsp[2][NUM_TEST]; + uint32_t w_ks_r_hit_sp[2][NUM_TEST]; + uint32_t w_ks_r_miss[2][NUM_TEST]; + uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST]; + uint32_t w_ks_r_hit_extbkt[2][NUM_TEST]; +}; + +static struct rwc_perf rwc_lf_results, rwc_non_lf_results; + +static struct { + uint32_t *keys; + uint32_t *keys_no_ks; + uint32_t *keys_ks; + uint32_t *keys_absent; + uint32_t *keys_shift_path; + uint32_t *keys_non_shift_path; + uint32_t *keys_ext_bkt; + uint32_t *keys_ks_extbkt; + uint32_t count_keys_no_ks; + uint32_t count_keys_ks; + uint32_t count_keys_absent; + uint32_t count_keys_shift_path; + uint32_t count_keys_non_shift_path; + uint32_t count_keys_extbkt; + uint32_t count_keys_ks_extbkt; + uint32_t single_insert; + struct rte_hash *h; +} tbl_rwc_test_param; + +static rte_atomic64_t gread_cycles; +static rte_atomic64_t greads; + +static volatile uint8_t writer_done; + +static uint16_t enabled_core_ids[RTE_MAX_LCORE]; + +static uint8_t *scanned_bkts; + +static inline uint16_t +get_short_sig(const hash_sig_t hash) +{ + return hash >> 16; +} + +static inline uint32_t +get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h, + const hash_sig_t hash) +{ + uint32_t num_buckets; + uint32_t bucket_bitmask; + num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; + bucket_bitmask = num_buckets - 1; + return hash & bucket_bitmask; +} + +static inline uint32_t +get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h, + uint32_t cur_bkt_idx, uint16_t sig) +{ + uint32_t num_buckets; + uint32_t bucket_bitmask; + num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; + bucket_bitmask = num_buckets - 1; + return (cur_bkt_idx ^ sig) & bucket_bitmask; +} + + +static inline int +get_enabled_cores_list(void) +{ + uint32_t i = 0; + uint16_t core_id; + uint32_t max_cores = rte_lcore_count(); + RTE_LCORE_FOREACH(core_id) { + enabled_core_ids[i] = core_id; + i++; + } + + if (i != max_cores) { + printf("Number of enabled cores in list is different from " + "number given by rte_lcore_count()\n"); + return -1; + } + return 0; +} + +static int +init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt) +{ + struct rte_hash *handle; + + struct rte_hash_parameters hash_params = { + .entries = TOTAL_ENTRY, + .key_len = sizeof(uint32_t), + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + + if (use_jhash) + hash_params.hash_func = rte_jhash; + else + hash_params.hash_func = rte_hash_crc; + + if (rwc_lf) + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF | + RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; + else if (htm) + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | + RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | + RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; + else + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY | + RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; + + if (ext_bkt) + hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE; + + hash_params.name = "tests"; + + handle = rte_hash_create(&hash_params); + if (handle == NULL) { + printf("hash creation failed"); + return -1; + } + + tbl_rwc_test_param.h = handle; + return 0; +} + +static inline int +check_bucket(uint32_t bkt_idx, uint32_t key) +{ + uint32_t iter; + uint32_t prev_iter; + uint32_t diff; + uint32_t count = 0; + const void *next_key; + void *next_data; + + /* Temporary bucket to hold the keys */ + uint32_t keys_in_bkt[8]; + + iter = bkt_idx * 8; + prev_iter = iter; + while (rte_hash_iterate(tbl_rwc_test_param.h, + &next_key, &next_data, &iter) >= 0) { + + /* Check for duplicate entries */ + if (*(const uint32_t *)next_key == key) + return 1; + + /* Identify if there is any free entry in the bucket */ + diff = iter - prev_iter; + if (diff > 1) + break; + + prev_iter = iter; + keys_in_bkt[count] = *(const uint32_t *)next_key; + count++; + + /* All entries in the bucket are occupied */ + if (count == 8) { + + /* + * Check if bucket was not scanned before, to avoid + * duplicate keys. + */ + if (scanned_bkts[bkt_idx] == 0) { + /* + * Since this bucket (pointed to by bkt_idx) is + * full, it is likely that key(s) in this + * bucket will be on the shift path, when + * collision occurs. Thus, add it to + * keys_shift_path. + */ + memcpy(tbl_rwc_test_param.keys_shift_path + + tbl_rwc_test_param.count_keys_shift_path + , keys_in_bkt, 32); + tbl_rwc_test_param.count_keys_shift_path += 8; + scanned_bkts[bkt_idx] = 1; + } + return -1; + } + } + return 0; +} + +static int +generate_keys(void) +{ + uint32_t *keys = NULL; + uint32_t *keys_no_ks = NULL; + uint32_t *keys_ks = NULL; + uint32_t *keys_absent = NULL; + uint32_t *keys_non_shift_path = NULL; + uint32_t *keys_ext_bkt = NULL; + uint32_t *keys_ks_extbkt = NULL; + uint32_t *found = NULL; + uint32_t count_keys_no_ks = 0; + uint32_t count_keys_ks = 0; + uint32_t count_keys_extbkt = 0; + uint32_t i; + + if (init_params(0, 0, 0, 0) != 0) + return -1; + + /* + * keys will consist of a) keys whose addition to the hash table + * will result in shifting of the existing keys to their alternate + * locations b) keys whose addition to the hash table will not result + * in shifting of the existing keys. + */ + keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (keys == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys that + * will NOT result in shifting of the existing keys to their alternate + * locations. Roughly around 900K keys. + */ + keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (keys_no_ks == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will + * result in shifting of the existing keys to their alternate locations. + * Roughly around 146K keys. There might be repeating keys. More code is + * required to filter out these keys which will complicate the test case + */ + keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (keys_ks == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* Used to identify keys not inserted in the hash table */ + found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (found == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * This consist of keys not inserted to the hash table. + * Used to test perf of lookup on keys that do not exist in the table. + */ + keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (keys_absent == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * This consist of keys which are likely to be on the shift + * path (i.e. being moved to alternate location), when collision occurs + * on addition of a key to an already full primary bucket. + * Used to test perf of lookup on keys that are on the shift path. + */ + tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) * + TOTAL_INSERT, 0); + if (tbl_rwc_test_param.keys_shift_path == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * This consist of keys which are never on the shift + * path (i.e. being moved to alternate location), when collision occurs + * on addition of a key to an already full primary bucket. + * Used to test perf of lookup on keys that are not on the shift path. + */ + keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, + 0); + if (keys_non_shift_path == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * This consist of keys which will be stored in extended buckets + */ + keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (keys_ext_bkt == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + /* + * This consist of keys which when deleted causes shifting of keys + * in extended buckets to respective secondary buckets + */ + keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0); + if (keys_ks_extbkt == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + hash_sig_t sig; + uint32_t prim_bucket_idx; + uint32_t sec_bucket_idx; + uint16_t short_sig; + uint32_t num_buckets; + num_buckets = rte_align32pow2(TOTAL_ENTRY) / 8; + int ret; + + /* + * Used to mark bkts in which at least one key was shifted to its + * alternate location + */ + scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0); + if (scanned_bkts == NULL) { + printf("RTE_MALLOC failed\n"); + goto err; + } + + tbl_rwc_test_param.keys = keys; + tbl_rwc_test_param.keys_no_ks = keys_no_ks; + tbl_rwc_test_param.keys_ks = keys_ks; + tbl_rwc_test_param.keys_absent = keys_absent; + tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path; + tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt; + tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt; + /* Generate keys by adding previous two keys, neglect overflow */ + printf("Generating keys...\n"); + keys[0] = 0; + keys[1] = 1; + for (i = 2; i < TOTAL_INSERT; i++) + keys[i] = keys[i-1] + keys[i-2]; + + /* Segregate keys into keys_no_ks and keys_ks */ + for (i = 0; i < TOTAL_INSERT; i++) { + /* Check if primary bucket has space.*/ + sig = rte_hash_hash(tbl_rwc_test_param.h, + tbl_rwc_test_param.keys+i); + prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h, + sig); + ret = check_bucket(prim_bucket_idx, keys[i]); + if (ret < 0) { + /* + * Primary bucket is full, this key will result in + * shifting of the keys to their alternate locations. + */ + keys_ks[count_keys_ks] = keys[i]; + count_keys_ks++; + } else if (ret == 0) { + /* + * Primary bucket has space, this key will not result in + * shifting of the keys. Hence, add key to the table. + */ + ret = rte_hash_add_key_data(tbl_rwc_test_param.h, + keys+i, + (void *)((uintptr_t)i)); + if (ret < 0) { + printf("writer failed %"PRIu32"\n", i); + break; + } + keys_no_ks[count_keys_no_ks] = keys[i]; + count_keys_no_ks++; + } + } + + for (i = 0; i < count_keys_no_ks; i++) { + /* + * Identify keys in keys_no_ks with value less than + * 4M (HTM enabled) OR 5K (HTM disabled) + */ + if (keys_no_ks[i] < TOTAL_INSERT) + found[keys_no_ks[i]]++; + } + + for (i = 0; i < count_keys_ks; i++) { + /* + * Identify keys in keys_ks with value less than + * 4M (HTM enabled) OR 5K (HTM disabled) + */ + if (keys_ks[i] < TOTAL_INSERT) + found[keys_ks[i]]++; + } + + uint32_t count_keys_absent = 0; + for (i = 0; i < TOTAL_INSERT; i++) { + /* + * Identify missing keys between 0 and + * 4M (HTM enabled) OR 5K (HTM disabled) + */ + if (found[i] == 0) + keys_absent[count_keys_absent++] = i; + } + + /* Find keys that will not be on the shift path */ + uint32_t iter; + const void *next_key; + void *next_data; + uint32_t count = 0; + for (i = 0; i < num_buckets; i++) { + /* Check bucket for no keys shifted to alternate locations */ + if (scanned_bkts[i] == 0) { + iter = i * 8; + while (rte_hash_iterate(tbl_rwc_test_param.h, + &next_key, &next_data, &iter) >= 0) { + + /* Check if key belongs to the current bucket */ + if (i >= (iter-1)/8) + keys_non_shift_path[count++] + = *(const uint32_t *)next_key; + else + break; + } + } + } + + tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks; + tbl_rwc_test_param.count_keys_ks = count_keys_ks; + tbl_rwc_test_param.count_keys_absent = count_keys_absent; + tbl_rwc_test_param.count_keys_non_shift_path = count; + + memset(scanned_bkts, 0, num_buckets); + count = 0; + /* Find keys that will be in extended buckets */ + for (i = 0; i < count_keys_ks; i++) { + ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i); + if (ret < 0) { + /* Key will be added to ext bkt */ + keys_ext_bkt[count_keys_extbkt++] = keys_ks[i]; + /* Sec bkt to be added to keys_ks_extbkt */ + sig = rte_hash_hash(tbl_rwc_test_param.h, + tbl_rwc_test_param.keys_ks + i); + prim_bucket_idx = get_prim_bucket_index( + tbl_rwc_test_param.h, sig); + short_sig = get_short_sig(sig); + sec_bucket_idx = get_alt_bucket_index( + tbl_rwc_test_param.h, + prim_bucket_idx, short_sig); + if (scanned_bkts[sec_bucket_idx] == 0) + scanned_bkts[sec_bucket_idx] = 1; + } + } + + /* Find keys that will shift keys in ext bucket*/ + for (i = 0; i < num_buckets; i++) { + if (scanned_bkts[i] == 1) { + iter = i * 8; + while (rte_hash_iterate(tbl_rwc_test_param.h, + &next_key, &next_data, &iter) >= 0) { + /* Check if key belongs to the current bucket */ + if (i >= (iter-1)/8) + keys_ks_extbkt[count++] + = *(const uint32_t *)next_key; + else + break; + } + } + } + + tbl_rwc_test_param.count_keys_ks_extbkt = count; + tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt; + + printf("\nCount of keys NOT causing shifting of existing keys to " + "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks); + printf("\nCount of keys causing shifting of existing keys to alternate " + "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks); + printf("Count of absent keys that will never be added to the hash " + "table: %d\n\n", tbl_rwc_test_param.count_keys_absent); + printf("Count of keys likely to be on the shift path: %d\n\n", + tbl_rwc_test_param.count_keys_shift_path); + printf("Count of keys not likely to be on the shift path: %d\n\n", + tbl_rwc_test_param.count_keys_non_shift_path); + printf("Count of keys in extended buckets: %d\n\n", + tbl_rwc_test_param.count_keys_extbkt); + printf("Count of keys shifting keys in ext buckets: %d\n\n", + tbl_rwc_test_param.count_keys_ks_extbkt); + + rte_free(found); + rte_free(scanned_bkts); + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_free(keys); + rte_free(keys_no_ks); + rte_free(keys_ks); + rte_free(keys_absent); + rte_free(found); + rte_free(tbl_rwc_test_param.keys_shift_path); + rte_free(keys_non_shift_path); + rte_free(keys_ext_bkt); + rte_free(keys_ks_extbkt); + rte_free(scanned_bkts); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +static int +test_rwc_reader(__attribute__((unused)) void *arg) +{ + uint32_t i, j; + int ret; + uint64_t begin, cycles; + uint32_t loop_cnt = 0; + uint8_t read_type = (uint8_t)((uintptr_t)arg); + uint32_t read_cnt; + uint32_t *keys; + uint32_t extra_keys; + int32_t pos[BULK_LOOKUP_SIZE]; + void *temp_a[BULK_LOOKUP_SIZE]; + + if (read_type & READ_FAIL) { + keys = tbl_rwc_test_param.keys_absent; + read_cnt = tbl_rwc_test_param.count_keys_absent; + } else if (read_type & READ_PASS_NO_KEY_SHIFTS) { + keys = tbl_rwc_test_param.keys_no_ks; + read_cnt = tbl_rwc_test_param.count_keys_no_ks; + } else if (read_type & READ_PASS_SHIFT_PATH) { + keys = tbl_rwc_test_param.keys_shift_path; + read_cnt = tbl_rwc_test_param.count_keys_shift_path; + } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) { + keys = tbl_rwc_test_param.keys_ext_bkt; + read_cnt = tbl_rwc_test_param.count_keys_extbkt; + } else { + keys = tbl_rwc_test_param.keys_non_shift_path; + read_cnt = tbl_rwc_test_param.count_keys_non_shift_path; + } + + extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1); + + begin = rte_rdtsc_precise(); + do { + if (read_type & BULK_LOOKUP) { + for (i = 0; i < (read_cnt - extra_keys); + i += BULK_LOOKUP_SIZE) { + /* Array of pointer to the list of keys */ + for (j = 0; j < BULK_LOOKUP_SIZE; j++) + temp_a[j] = keys + i + j; + rte_hash_lookup_bulk(tbl_rwc_test_param.h, + (const void **) + ((uintptr_t)temp_a), + BULK_LOOKUP_SIZE, pos); + /* Validate lookup result */ + for (j = 0; j < BULK_LOOKUP_SIZE; j++) + if ((read_type & READ_FAIL && + pos[j] != -ENOENT) || + (!(read_type & READ_FAIL) && + pos[j] == -ENOENT)) { + printf("lookup failed!" + "%"PRIu32"\n", + keys[i + j]); + return -1; + } + } + for (j = 0; j < extra_keys; j++) + temp_a[j] = keys + i + j; + + rte_hash_lookup_bulk(tbl_rwc_test_param.h, + (const void **) + ((uintptr_t)temp_a), + extra_keys, pos); + for (j = 0; j < extra_keys; j++) + if ((read_type & READ_FAIL && + pos[j] != -ENOENT) || + (!(read_type & READ_FAIL) && + pos[j] == -ENOENT)) { + printf("lookup failed! %"PRIu32"\n", + keys[i + j]); + return -1; + } + } else { + for (i = 0; i < read_cnt; i++) { + ret = rte_hash_lookup + (tbl_rwc_test_param.h, keys + i); + if (((read_type & READ_FAIL) && + (ret != -ENOENT)) || + (!(read_type & READ_FAIL) && + ret == -ENOENT)) { + printf("lookup failed! %"PRIu32"\n", + keys[i]); + return -1; + } + } + } + loop_cnt++; + } while (!writer_done); + + cycles = rte_rdtsc_precise() - begin; + rte_atomic64_add(&gread_cycles, cycles); + rte_atomic64_add(&greads, read_cnt*loop_cnt); + return 0; +} + +static int +write_keys(uint8_t write_type) +{ + uint32_t i; + int ret; + uint32_t key_cnt = 0; + uint32_t *keys; + if (write_type == WRITE_KEY_SHIFT) { + key_cnt = tbl_rwc_test_param.count_keys_ks; + keys = tbl_rwc_test_param.keys_ks; + } else if (write_type == WRITE_NO_KEY_SHIFT) { + key_cnt = tbl_rwc_test_param.count_keys_no_ks; + keys = tbl_rwc_test_param.keys_no_ks; + } else if (write_type == WRITE_EXT_BKT) { + key_cnt = tbl_rwc_test_param.count_keys_extbkt; + keys = tbl_rwc_test_param.keys_ext_bkt; + } + for (i = 0; i < key_cnt; i++) { + ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i); + if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) { + printf("writer failed %"PRIu32"\n", i); + return -1; + } + } + return 0; +} + +static int +test_rwc_multi_writer(__attribute__((unused)) void *arg) +{ + uint32_t i, offset; + uint32_t pos_core = (uint32_t)((uintptr_t)arg); + offset = pos_core * tbl_rwc_test_param.single_insert; + for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) + rte_hash_add_key(tbl_rwc_test_param.h, + tbl_rwc_test_param.keys_ks + i); + return 0; +} + +/* + * Test lookup perf: + * Reader(s) lookup keys present in the table. + */ +static int +test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf, + int htm, int ext_bkt) +{ + unsigned int n, m; + uint64_t i; + int use_jhash = 0; + uint8_t write_type = WRITE_NO_KEY_SHIFT; + uint8_t read_type = READ_PASS_NO_KEY_SHIFTS; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Hash add - no key-shifts, read - hit\n"); + for (m = 0; m < 2; m++) { + if (m == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < rwc_core_cnt[n] + 1) + goto finish; + + printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + writer_done = 0; + if (write_keys(write_type) < 0) + goto err; + writer_done = 1; + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + rwc_perf_results->w_no_ks_r_hit[m][n] + = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", cycles_per_lookup); + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +/* + * Test lookup perf: + * Reader(s) lookup keys absent in the table while + * 'Main' thread adds with no key-shifts. + */ +static int +test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, + int htm, int ext_bkt) +{ + unsigned int n, m; + uint64_t i; + int use_jhash = 0; + uint8_t write_type = WRITE_NO_KEY_SHIFT; + uint8_t read_type = READ_FAIL; + int ret; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n"); + for (m = 0; m < 2; m++) { + if (m == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < rwc_core_cnt[n] + 1) + goto finish; + + printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + writer_done = 0; + + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + ret = write_keys(write_type); + writer_done = 1; + + if (ret < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + rwc_perf_results->w_no_ks_r_miss[m][n] + = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", cycles_per_lookup); + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +/* + * Test lookup perf: + * Reader(s) lookup keys present in the table and not likely to be on the + * shift path while 'Main' thread adds keys causing key-shifts. + */ +static int +test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results, + int rwc_lf, int htm, int ext_bkt) +{ + unsigned int n, m; + uint64_t i; + int use_jhash = 0; + int ret; + uint8_t write_type; + uint8_t read_type = READ_PASS_NON_SHIFT_PATH; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Hash add - key shift, Hash lookup - hit" + " (non-shift-path)\n"); + for (m = 0; m < 2; m++) { + if (m == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < rwc_core_cnt[n] + 1) + goto finish; + + printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + writer_done = 0; + write_type = WRITE_NO_KEY_SHIFT; + if (write_keys(write_type) < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + write_type = WRITE_KEY_SHIFT; + ret = write_keys(write_type); + writer_done = 1; + + if (ret < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + rwc_perf_results->w_ks_r_hit_nsp[m][n] + = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", cycles_per_lookup); + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +/* + * Test lookup perf: + * Reader(s) lookup keys present in the table and likely on the shift-path while + * 'Main' thread adds keys causing key-shifts. + */ +static int +test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf, + int htm, int ext_bkt) +{ + unsigned int n, m; + uint64_t i; + int use_jhash = 0; + int ret; + uint8_t write_type; + uint8_t read_type = READ_PASS_SHIFT_PATH; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)" + "\n"); + + for (m = 0; m < 2; m++) { + if (m == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < rwc_core_cnt[n] + 1) + goto finish; + + printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + writer_done = 0; + write_type = WRITE_NO_KEY_SHIFT; + if (write_keys(write_type) < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + write_type = WRITE_KEY_SHIFT; + ret = write_keys(write_type); + writer_done = 1; + + if (ret < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + rwc_perf_results->w_ks_r_hit_sp[m][n] + = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", cycles_per_lookup); + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +/* + * Test lookup perf: + * Reader(s) lookup keys absent in the table while + * 'Main' thread adds keys causing key-shifts. + */ +static int +test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int + htm, int ext_bkt) +{ + unsigned int n, m; + uint64_t i; + int use_jhash = 0; + int ret; + uint8_t write_type; + uint8_t read_type = READ_FAIL; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Hash add - key shift, Hash lookup - miss\n"); + for (m = 0; m < 2; m++) { + if (m == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < rwc_core_cnt[n] + 1) + goto finish; + + printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + writer_done = 0; + write_type = WRITE_NO_KEY_SHIFT; + if (write_keys(write_type) < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + write_type = WRITE_KEY_SHIFT; + ret = write_keys(write_type); + writer_done = 1; + + if (ret < 0) + goto err; + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", cycles_per_lookup); + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +/* + * Test lookup perf for multi-writer: + * Reader(s) lookup keys present in the table and likely on the shift-path while + * Writers add keys causing key-shiftsi. + * Writers are running in parallel, on different data plane cores. + */ +static int +test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf, + int htm, int ext_bkt) +{ + unsigned int n, m, k; + uint64_t i; + int use_jhash = 0; + uint8_t write_type; + uint8_t read_type = READ_PASS_SHIFT_PATH; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Multi-add-lookup\n"); + uint8_t pos_core; + for (m = 1; m < NUM_TEST; m++) { + /* Calculate keys added by each writer */ + tbl_rwc_test_param.single_insert = + tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m]; + for (k = 0; k < 2; k++) { + if (k == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < (rwc_core_cnt[n] + + rwc_core_cnt[m] + 1)) + goto finish; + + printf("\nNumber of writers: %u", + rwc_core_cnt[m]); + printf("\nNumber of readers: %u\n", + rwc_core_cnt[n]); + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + writer_done = 0; + write_type = WRITE_NO_KEY_SHIFT; + if (write_keys(write_type) < 0) + goto err; + + /* Launch reader(s) */ + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + write_type = WRITE_KEY_SHIFT; + pos_core = 0; + + /* Launch writers */ + for (; i <= rwc_core_cnt[m] + + rwc_core_cnt[n]; i++) { + rte_eal_remote_launch + (test_rwc_multi_writer, + (void *)(uintptr_t)pos_core, + enabled_core_ids[i]); + pos_core++; + } + + /* Wait for writers to complete */ + for (i = rwc_core_cnt[n] + 1; + i <= rwc_core_cnt[m] + rwc_core_cnt[n]; + i++) + rte_eal_wait_lcore(enabled_core_ids[i]); + + writer_done = 1; + + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) + / rte_atomic64_read(&greads); + rwc_perf_results->multi_rw[m][k][n] + = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", + cycles_per_lookup); + } + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +/* + * Test lookup perf: + * Reader(s) lookup keys present in the extendable bkt. + */ +static int +test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results, + int rwc_lf, int htm, int ext_bkt) +{ + unsigned int n, m; + uint64_t i; + int use_jhash = 0; + uint8_t write_type; + uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gread_cycles); + + if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0) + goto err; + printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n"); + for (m = 0; m < 2; m++) { + if (m == 1) { + printf("\n** With bulk-lookup **\n"); + read_type |= BULK_LOOKUP; + } + for (n = 0; n < NUM_TEST; n++) { + unsigned int tot_lcore = rte_lcore_count(); + if (tot_lcore < rwc_core_cnt[n] + 1) + goto finish; + + printf("\nNumber of readers: %u\n", rwc_core_cnt[n]); + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + + rte_hash_reset(tbl_rwc_test_param.h); + write_type = WRITE_NO_KEY_SHIFT; + if (write_keys(write_type) < 0) + goto err; + write_type = WRITE_KEY_SHIFT; + if (write_keys(write_type) < 0) + goto err; + writer_done = 0; + for (i = 1; i <= rwc_core_cnt[n]; i++) + rte_eal_remote_launch(test_rwc_reader, + (void *)(uintptr_t)read_type, + enabled_core_ids[i]); + for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt; + i++) { + if (rte_hash_del_key(tbl_rwc_test_param.h, + tbl_rwc_test_param.keys_ks_extbkt + i) + < 0) { + printf("Delete Failed: %u\n", + tbl_rwc_test_param.keys_ks_extbkt[i]); + goto err; + } + } + writer_done = 1; + + for (i = 1; i <= rwc_core_cnt[n]; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto err; + + unsigned long long cycles_per_lookup = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + rwc_perf_results->w_ks_r_hit_extbkt[m][n] + = cycles_per_lookup; + printf("Cycles per lookup: %llu\n", cycles_per_lookup); + } + } + +finish: + rte_hash_free(tbl_rwc_test_param.h); + return 0; + +err: + rte_eal_mp_wait_lcore(); + rte_hash_free(tbl_rwc_test_param.h); + return -1; +} + +static int +test_hash_readwrite_lf_perf_main(void) +{ + /* + * Variables used to choose different tests. + * rwc_lf indicates if read-write concurrency lock-free support is + * enabled. + * htm indicates if Hardware transactional memory support is enabled. + */ + int rwc_lf = 0; + int htm; + int ext_bkt = 0; + + if (rte_lcore_count() < 2) { + printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n"); + return TEST_SKIPPED; + } + + setlocale(LC_NUMERIC, ""); + + /* Reset tbl_rwc_test_param to discard values from previous run */ + memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param)); + + if (rte_tm_supported()) + htm = 1; + else + htm = 0; + + if (generate_keys() != 0) + return -1; + if (get_enabled_cores_list() != 0) + return -1; + + if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) { + rwc_lf = 1; + ext_bkt = 1; + printf("Test lookup with read-write concurrency lock free support" + " enabled\n"); + if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; + if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; + } + printf("\nTest lookup with read-write concurrency lock free support" + " disabled\n"); + rwc_lf = 0; + if (!htm) { + printf("With HTM Disabled\n"); + if (!RUN_WITH_HTM_DISABLED) { + printf("Enable RUN_WITH_HTM_DISABLED to test with" + " lock-free disabled"); + goto results; + } + } else + printf("With HTM Enabled\n"); + if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm, + ext_bkt) < 0) + return -1; + if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf, + htm, ext_bkt) < 0) + return -1; +results: + printf("\n\t\t\t\t\t\t********** Results summary **********\n\n"); + int i, j, k; + for (j = 0; j < 2; j++) { + if (j == 1) + printf("\n\t\t\t\t\t#######********** Bulk Lookup " + "**********#######\n\n"); + printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t" + "\t\t\t\t_________________\n"); + printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t" + "\t\t\tCycles per lookup\n"); + printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t" + "\t\t\t_________________\n"); + for (i = 0; i < NUM_TEST; i++) { + printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]); + printf("Enabled\t\t"); + printf("N/A\t\t"); + printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" + "%u\n\t\t\t\t\t\t\t\t", + rwc_lf_results.w_no_ks_r_hit[j][i]); + printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" + "%u\n\t\t\t\t\t\t\t\t", + rwc_lf_results.w_no_ks_r_miss[j][i]); + printf("Hash add - key-shifts, lookup - hit" + "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", + rwc_lf_results.w_ks_r_hit_nsp[j][i]); + printf("Hash add - key-shifts, lookup - hit " + "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", + rwc_lf_results.w_ks_r_hit_sp[j][i]); + printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" + "%u\n\t\t\t\t\t\t\t\t", + rwc_lf_results.w_ks_r_miss[j][i]); + printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" + "%u\n\n\t\t\t\t", + rwc_lf_results.w_ks_r_hit_extbkt[j][i]); + + printf("Disabled\t"); + if (htm) + printf("Enabled\t\t"); + else + printf("Disabled\t"); + printf("Hash add - no key-shifts, lookup - hit\t\t\t\t" + "%u\n\t\t\t\t\t\t\t\t", + rwc_non_lf_results.w_no_ks_r_hit[j][i]); + printf("Hash add - no key-shifts, lookup - miss\t\t\t\t" + "%u\n\t\t\t\t\t\t\t\t", + rwc_non_lf_results.w_no_ks_r_miss[j][i]); + printf("Hash add - key-shifts, lookup - hit " + "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", + rwc_non_lf_results.w_ks_r_hit_nsp[j][i]); + printf("Hash add - key-shifts, lookup - hit " + "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t", + rwc_non_lf_results.w_ks_r_hit_sp[j][i]); + printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t" + "%u\n\t\t\t\t\t\t\t\t", + rwc_non_lf_results.w_ks_r_miss[j][i]); + printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t" + "%u\n", + rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]); + + printf("_______\t\t_______\t\t_________\t___\t\t" + "_________\t\t\t\t\t\t_________________\n"); + } + + for (i = 1; i < NUM_TEST; i++) { + for (k = 0; k < NUM_TEST; k++) { + printf("%u", rwc_core_cnt[i]); + printf("\t\t%u\t\t", rwc_core_cnt[k]); + printf("Enabled\t\t"); + printf("N/A\t\t"); + printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t" + "\t\t", + rwc_lf_results.multi_rw[i][j][k]); + printf("Disabled\t"); + if (htm) + printf("Enabled\t\t"); + else + printf("Disabled\t"); + printf("Multi-add-lookup\t\t\t\t\t\t%u\n", + rwc_non_lf_results.multi_rw[i][j][k]); + + printf("_______\t\t_______\t\t_________\t___" + "\t\t_________\t\t\t\t\t\t" + "_________________\n"); + } + } + } + rte_free(tbl_rwc_test_param.keys); + rte_free(tbl_rwc_test_param.keys_no_ks); + rte_free(tbl_rwc_test_param.keys_ks); + rte_free(tbl_rwc_test_param.keys_absent); + rte_free(tbl_rwc_test_param.keys_shift_path); + rte_free(tbl_rwc_test_param.keys_non_shift_path); + rte_free(tbl_rwc_test_param.keys_ext_bkt); + rte_free(tbl_rwc_test_param.keys_ks_extbkt); + return 0; +} + +REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest, + test_hash_readwrite_lf_perf_main);