app/test/test_hash_readwrite_lf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Arm Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
  20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
  21 #endif
  22
  23 #define BULK_LOOKUP_SIZE 32
  24
  25 #define RUN_WITH_HTM_DISABLED 0
  26
  27 #if (RUN_WITH_HTM_DISABLED)
  28
  29 #define TOTAL_ENTRY (5*1024)
  30 #define TOTAL_INSERT (5*1024)
  31
  32 #else
  33
  34 #define TOTAL_ENTRY (4*1024*1024)
  35 #define TOTAL_INSERT (4*1024*1024)
  36
  37 #endif
  38
  39 #define READ_FAIL 1
  40 #define READ_PASS_NO_KEY_SHIFTS 2
  41 #define READ_PASS_SHIFT_PATH 4
  42 #define READ_PASS_NON_SHIFT_PATH 8
  43 #define BULK_LOOKUP 16
  44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
  45
  46 #define WRITE_NO_KEY_SHIFT 0
  47 #define WRITE_KEY_SHIFT 1
  48 #define WRITE_EXT_BKT 2
  49
  50 #define NUM_TEST 3
  51 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
  52
  53 struct rwc_perf {
  54         uint32_t w_no_ks_r_hit[2][NUM_TEST];
  55         uint32_t w_no_ks_r_miss[2][NUM_TEST];
  56         uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
  57         uint32_t w_ks_r_hit_sp[2][NUM_TEST];
  58         uint32_t w_ks_r_miss[2][NUM_TEST];
  59         uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
  60         uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
  61 };
  62
  63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
  64
  65 struct {
  66         uint32_t *keys;
  67         uint32_t *keys_no_ks;
  68         uint32_t *keys_ks;
  69         uint32_t *keys_absent;
  70         uint32_t *keys_shift_path;
  71         uint32_t *keys_non_shift_path;
  72         uint32_t *keys_ext_bkt;
  73         uint32_t *keys_ks_extbkt;
  74         uint32_t count_keys_no_ks;
  75         uint32_t count_keys_ks;
  76         uint32_t count_keys_absent;
  77         uint32_t count_keys_shift_path;
  78         uint32_t count_keys_non_shift_path;
  79         uint32_t count_keys_extbkt;
  80         uint32_t count_keys_ks_extbkt;
  81         uint32_t single_insert;
  82         struct rte_hash *h;
  83 } tbl_rwc_test_param;
  84
  85 static rte_atomic64_t gread_cycles;
  86 static rte_atomic64_t greads;
  87
  88 static volatile uint8_t writer_done;
  89
  90 uint16_t enabled_core_ids[RTE_MAX_LCORE];
  91
  92 uint8_t *scanned_bkts;
  93
  94 static inline uint16_t
  95 get_short_sig(const hash_sig_t hash)
  96 {
  97         return hash >> 16;
  98 }
  99
 100 static inline uint32_t
 101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 102                       const hash_sig_t hash)
 103 {
 104         uint32_t num_buckets;
 105         uint32_t bucket_bitmask;
 106         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 107         bucket_bitmask = num_buckets - 1;
 108         return hash & bucket_bitmask;
 109 }
 110
 111 static inline uint32_t
 112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 113                         uint32_t cur_bkt_idx, uint16_t sig)
 114 {
 115         uint32_t num_buckets;
 116         uint32_t bucket_bitmask;
 117         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 118         bucket_bitmask = num_buckets - 1;
 119         return (cur_bkt_idx ^ sig) & bucket_bitmask;
 120 }
 121
 122
 123 static inline int
 124 get_enabled_cores_list(void)
 125 {
 126         uint32_t i = 0;
 127         uint16_t core_id;
 128         uint32_t max_cores = rte_lcore_count();
 129         for (core_id = 0; core_id < RTE_MAX_LCORE && i < max_cores; core_id++) {
 130                 if (rte_lcore_is_enabled(core_id)) {
 131                         enabled_core_ids[i] = core_id;
 132                         i++;
 133                 }
 134         }
 135
 136         if (i != max_cores) {
 137                 printf("Number of enabled cores in list is different from "
 138                         "number given by rte_lcore_count()\n");
 139                 return -1;
 140         }
 141         return 0;
 142 }
 143
 144 static inline int
 145 check_bucket(uint32_t bkt_idx, uint32_t key)
 146 {
 147         uint32_t iter;
 148         uint32_t prev_iter;
 149         uint32_t diff;
 150         uint32_t count = 0;
 151         const void *next_key;
 152         void *next_data;
 153
 154         /* Temporary bucket to hold the keys */
 155         uint32_t keys_in_bkt[8];
 156
 157         iter = bkt_idx * 8;
 158         prev_iter = iter;
 159         while (rte_hash_iterate(tbl_rwc_test_param.h,
 160                         &next_key, &next_data, &iter) >= 0) {
 161
 162                 /* Check for duplicate entries */
 163                 if (*(const uint32_t *)next_key == key)
 164                         return 1;
 165
 166                 /* Identify if there is any free entry in the bucket */
 167                 diff = iter - prev_iter;
 168                 if (diff > 1)
 169                         break;
 170
 171                 prev_iter = iter;
 172                 keys_in_bkt[count] = *(const uint32_t *)next_key;
 173                 count++;
 174
 175                 /* All entries in the bucket are occupied */
 176                 if (count == 8) {
 177
 178                         /*
 179                          * Check if bucket was not scanned before, to avoid
 180                          * duplicate keys.
 181                          */
 182                         if (scanned_bkts[bkt_idx] == 0) {
 183                                 /*
 184                                  * Since this bucket (pointed to by bkt_idx) is
 185                                  * full, it is likely that key(s) in this
 186                                  * bucket will be on the shift path, when
 187                                  * collision occurs. Thus, add it to
 188                                  * keys_shift_path.
 189                                  */
 190                                 memcpy(tbl_rwc_test_param.keys_shift_path +
 191                                         tbl_rwc_test_param.count_keys_shift_path
 192                                         , keys_in_bkt, 32);
 193                                 tbl_rwc_test_param.count_keys_shift_path += 8;
 194                                 scanned_bkts[bkt_idx] = 1;
 195                         }
 196                         return -1;
 197                 }
 198         }
 199         return 0;
 200 }
 201
 202 static int
 203 generate_keys(void)
 204 {
 205         uint32_t *keys = NULL;
 206         uint32_t *keys_no_ks = NULL;
 207         uint32_t *keys_ks = NULL;
 208         uint32_t *keys_absent = NULL;
 209         uint32_t *keys_non_shift_path = NULL;
 210         uint32_t *keys_ext_bkt = NULL;
 211         uint32_t *keys_ks_extbkt = NULL;
 212         uint32_t *found = NULL;
 213         uint32_t count_keys_no_ks = 0;
 214         uint32_t count_keys_ks = 0;
 215         uint32_t count_keys_extbkt = 0;
 216         uint32_t i;
 217
 218         /*
 219          * keys will consist of a) keys whose addition to the hash table
 220          * will result in shifting of the existing keys to their alternate
 221          * locations b) keys whose addition to the hash table will not result
 222          * in shifting of the existing keys.
 223          */
 224         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 225         if (keys == NULL) {
 226                 printf("RTE_MALLOC failed\n");
 227                 goto err;
 228         }
 229
 230         /*
 231          * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
 232          * will NOT result in shifting of the existing keys to their alternate
 233          * locations. Roughly around 900K keys.
 234          */
 235         keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 236         if (keys_no_ks == NULL) {
 237                 printf("RTE_MALLOC failed\n");
 238                 goto err;
 239         }
 240
 241         /*
 242          * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
 243          * result in shifting of the existing keys to their alternate locations.
 244          * Roughly around 146K keys. There might be repeating keys. More code is
 245          * required to filter out these keys which will complicate the test case
 246          */
 247         keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 248         if (keys_ks == NULL) {
 249                 printf("RTE_MALLOC failed\n");
 250                 goto err;
 251         }
 252
 253         /* Used to identify keys not inserted in the hash table */
 254         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 255         if (found == NULL) {
 256                 printf("RTE_MALLOC failed\n");
 257                 goto err;
 258         }
 259
 260         /*
 261          * This consist of keys not inserted to the hash table.
 262          * Used to test perf of lookup on keys that do not exist in the table.
 263          */
 264         keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 265         if (keys_absent == NULL) {
 266                 printf("RTE_MALLOC failed\n");
 267                 goto err;
 268         }
 269
 270         /*
 271          * This consist of keys which are likely to be on the shift
 272          * path (i.e. being moved to alternate location), when collision occurs
 273          * on addition of a key to an already full primary bucket.
 274          * Used to test perf of lookup on keys that are on the shift path.
 275          */
 276         tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
 277                                                         TOTAL_INSERT, 0);
 278         if (tbl_rwc_test_param.keys_shift_path == NULL) {
 279                 printf("RTE_MALLOC failed\n");
 280                 goto err;
 281         }
 282
 283         /*
 284          * This consist of keys which are never on the shift
 285          * path (i.e. being moved to alternate location), when collision occurs
 286          * on addition of a key to an already full primary bucket.
 287          * Used to test perf of lookup on keys that are not on the shift path.
 288          */
 289         keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
 290                                          0);
 291         if (keys_non_shift_path == NULL) {
 292                 printf("RTE_MALLOC failed\n");
 293                 goto err;
 294         }
 295
 296         /*
 297          * This consist of keys which will be stored in extended buckets
 298          */
 299         keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 300         if (keys_ext_bkt == NULL) {
 301                 printf("RTE_MALLOC failed\n");
 302                 goto err;
 303         }
 304
 305         /*
 306          * This consist of keys which when deleted causes shifting of keys
 307          * in extended buckets to respective secondary buckets
 308          */
 309         keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 310         if (keys_ks_extbkt == NULL) {
 311                 printf("RTE_MALLOC failed\n");
 312                 goto err;
 313         }
 314
 315         hash_sig_t sig;
 316         uint32_t prim_bucket_idx;
 317         uint32_t sec_bucket_idx;
 318         uint16_t short_sig;
 319         uint32_t num_buckets;
 320         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 321         int ret;
 322
 323         /*
 324          * Used to mark bkts in which at least one key was shifted to its
 325          * alternate location
 326          */
 327         scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
 328         if (scanned_bkts == NULL) {
 329                 printf("RTE_MALLOC failed\n");
 330                 goto err;
 331         }
 332
 333         tbl_rwc_test_param.keys = keys;
 334         tbl_rwc_test_param.keys_no_ks = keys_no_ks;
 335         tbl_rwc_test_param.keys_ks = keys_ks;
 336         tbl_rwc_test_param.keys_absent = keys_absent;
 337         tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
 338         tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
 339         tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
 340         /* Generate keys by adding previous two keys, neglect overflow */
 341         printf("Generating keys...\n");
 342         keys[0] = 0;
 343         keys[1] = 1;
 344         for (i = 2; i < TOTAL_INSERT; i++)
 345                 keys[i] = keys[i-1] + keys[i-2];
 346
 347         /* Segregate keys into keys_no_ks and keys_ks */
 348         for (i = 0; i < TOTAL_INSERT; i++) {
 349                 /* Check if primary bucket has space.*/
 350                 sig = rte_hash_hash(tbl_rwc_test_param.h,
 351                                         tbl_rwc_test_param.keys+i);
 352                 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
 353                                                         sig);
 354                 ret = check_bucket(prim_bucket_idx, keys[i]);
 355                 if (ret < 0) {
 356                         /*
 357                          * Primary bucket is full, this key will result in
 358                          * shifting of the keys to their alternate locations.
 359                          */
 360                         keys_ks[count_keys_ks] = keys[i];
 361                         count_keys_ks++;
 362                 } else if (ret == 0) {
 363                         /*
 364                          * Primary bucket has space, this key will not result in
 365                          * shifting of the keys. Hence, add key to the table.
 366                          */
 367                         ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
 368                                                         keys+i,
 369                                                         (void *)((uintptr_t)i));
 370                         if (ret < 0) {
 371                                 printf("writer failed %"PRIu32"\n", i);
 372                                 break;
 373                         }
 374                         keys_no_ks[count_keys_no_ks] = keys[i];
 375                         count_keys_no_ks++;
 376                 }
 377         }
 378
 379         for (i = 0; i < count_keys_no_ks; i++) {
 380                 /*
 381                  * Identify keys in keys_no_ks with value less than
 382                  * 4M (HTM enabled) OR 5K (HTM disabled)
 383                  */
 384                 if (keys_no_ks[i] < TOTAL_INSERT)
 385                         found[keys_no_ks[i]]++;
 386         }
 387
 388         for (i = 0; i < count_keys_ks; i++) {
 389                 /*
 390                  * Identify keys in keys_ks with value less than
 391                  * 4M (HTM enabled) OR 5K (HTM disabled)
 392                  */
 393                 if (keys_ks[i] < TOTAL_INSERT)
 394                         found[keys_ks[i]]++;
 395         }
 396
 397         uint32_t count_keys_absent = 0;
 398         for (i = 0; i < TOTAL_INSERT; i++) {
 399                 /*
 400                  * Identify missing keys between 0 and
 401                  * 4M (HTM enabled) OR 5K (HTM disabled)
 402                  */
 403                 if (found[i] == 0)
 404                         keys_absent[count_keys_absent++] = i;
 405         }
 406
 407         /* Find keys that will not be on the shift path */
 408         uint32_t iter;
 409         const void *next_key;
 410         void *next_data;
 411         uint32_t count = 0;
 412         for (i = 0; i < num_buckets; i++) {
 413                 /* Check bucket for no keys shifted to alternate locations */
 414                 if (scanned_bkts[i] == 0) {
 415                         iter = i * 8;
 416                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 417                                 &next_key, &next_data, &iter) >= 0) {
 418
 419                                 /* Check if key belongs to the current bucket */
 420                                 if (i >= (iter-1)/8)
 421                                         keys_non_shift_path[count++]
 422                                                 = *(const uint32_t *)next_key;
 423                                 else
 424                                         break;
 425                         }
 426                 }
 427         }
 428
 429         tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
 430         tbl_rwc_test_param.count_keys_ks = count_keys_ks;
 431         tbl_rwc_test_param.count_keys_absent = count_keys_absent;
 432         tbl_rwc_test_param.count_keys_non_shift_path = count;
 433
 434         memset(scanned_bkts, 0, num_buckets);
 435         count = 0;
 436         /* Find keys that will be in extended buckets */
 437         for (i = 0; i < count_keys_ks; i++) {
 438                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
 439                 if (ret < 0) {
 440                         /* Key will be added to ext bkt */
 441                         keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
 442                         /* Sec bkt to be added to keys_ks_extbkt */
 443                         sig = rte_hash_hash(tbl_rwc_test_param.h,
 444                                         tbl_rwc_test_param.keys_ks + i);
 445                         prim_bucket_idx = get_prim_bucket_index(
 446                                                 tbl_rwc_test_param.h, sig);
 447                         short_sig = get_short_sig(sig);
 448                         sec_bucket_idx = get_alt_bucket_index(
 449                                                 tbl_rwc_test_param.h,
 450                                                 prim_bucket_idx, short_sig);
 451                         if (scanned_bkts[sec_bucket_idx] == 0)
 452                                 scanned_bkts[sec_bucket_idx] = 1;
 453                 }
 454         }
 455
 456         /* Find keys that will shift keys in ext bucket*/
 457         for (i = 0; i < num_buckets; i++) {
 458                 if (scanned_bkts[i] == 1) {
 459                         iter = i * 8;
 460                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 461                                 &next_key, &next_data, &iter) >= 0) {
 462                                 /* Check if key belongs to the current bucket */
 463                                 if (i >= (iter-1)/8)
 464                                         keys_ks_extbkt[count++]
 465                                                 = *(const uint32_t *)next_key;
 466                                 else
 467                                         break;
 468                         }
 469                 }
 470         }
 471
 472         tbl_rwc_test_param.count_keys_ks_extbkt = count;
 473         tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
 474
 475         printf("\nCount of keys NOT causing shifting of existing keys to "
 476         "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
 477         printf("\nCount of keys causing shifting of existing keys to alternate "
 478                 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
 479         printf("Count of absent keys that will never be added to the hash "
 480                 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
 481         printf("Count of keys likely to be on the shift path: %d\n\n",
 482                tbl_rwc_test_param.count_keys_shift_path);
 483         printf("Count of keys not likely to be on the shift path: %d\n\n",
 484                tbl_rwc_test_param.count_keys_non_shift_path);
 485         printf("Count of keys in extended buckets: %d\n\n",
 486                tbl_rwc_test_param.count_keys_extbkt);
 487         printf("Count of keys shifting keys in ext buckets: %d\n\n",
 488                tbl_rwc_test_param.count_keys_ks_extbkt);
 489
 490         rte_free(found);
 491         rte_hash_free(tbl_rwc_test_param.h);
 492         return 0;
 493
 494 err:
 495         rte_free(keys);
 496         rte_free(keys_no_ks);
 497         rte_free(keys_ks);
 498         rte_free(keys_absent);
 499         rte_free(found);
 500         rte_free(tbl_rwc_test_param.keys_shift_path);
 501         rte_free(keys_ext_bkt);
 502         rte_free(keys_ks_extbkt);
 503         rte_free(scanned_bkts);
 504         return -1;
 505 }
 506
 507 static int
 508 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
 509 {
 510         struct rte_hash *handle;
 511
 512         struct rte_hash_parameters hash_params = {
 513                 .entries = TOTAL_ENTRY,
 514                 .key_len = sizeof(uint32_t),
 515                 .hash_func_init_val = 0,
 516                 .socket_id = rte_socket_id(),
 517         };
 518
 519         if (use_jhash)
 520                 hash_params.hash_func = rte_jhash;
 521         else
 522                 hash_params.hash_func = rte_hash_crc;
 523
 524         if (rwc_lf)
 525                 hash_params.extra_flag =
 526                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
 527                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 528         else if (htm)
 529                 hash_params.extra_flag =
 530                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 531                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 532                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 533         else
 534                 hash_params.extra_flag =
 535                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 536                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 537
 538         if (ext_bkt)
 539                 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
 540
 541         hash_params.name = "tests";
 542
 543         handle = rte_hash_create(&hash_params);
 544         if (handle == NULL) {
 545                 printf("hash creation failed");
 546                 return -1;
 547         }
 548
 549         tbl_rwc_test_param.h = handle;
 550         return 0;
 551 }
 552
 553 static int
 554 test_rwc_reader(__attribute__((unused)) void *arg)
 555 {
 556         uint32_t i, j;
 557         int ret;
 558         uint64_t begin, cycles;
 559         uint32_t loop_cnt = 0;
 560         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 561         uint32_t read_cnt;
 562         uint32_t *keys;
 563         uint32_t extra_keys;
 564         int32_t *pos;
 565         void *temp_a[BULK_LOOKUP_SIZE];
 566
 567         /* Used to identify keys not inserted in the hash table */
 568         pos = rte_malloc(NULL, sizeof(uint32_t) * BULK_LOOKUP_SIZE, 0);
 569         if (pos == NULL) {
 570                 printf("RTE_MALLOC failed\n");
 571                 return -1;
 572         }
 573
 574         if (read_type & READ_FAIL) {
 575                 keys = tbl_rwc_test_param.keys_absent;
 576                 read_cnt = tbl_rwc_test_param.count_keys_absent;
 577         } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
 578                 keys = tbl_rwc_test_param.keys_no_ks;
 579                 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
 580         } else if (read_type & READ_PASS_SHIFT_PATH) {
 581                 keys = tbl_rwc_test_param.keys_shift_path;
 582                 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
 583         } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
 584                 keys = tbl_rwc_test_param.keys_ext_bkt;
 585                 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
 586         } else {
 587                 keys = tbl_rwc_test_param.keys_non_shift_path;
 588                 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
 589         }
 590
 591         extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
 592
 593         begin = rte_rdtsc_precise();
 594         do {
 595                 if (read_type & BULK_LOOKUP) {
 596                         for (i = 0; i < (read_cnt - extra_keys);
 597                              i += BULK_LOOKUP_SIZE) {
 598                                 /* Array of  pointer to the list of keys */
 599                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 600                                         temp_a[j] = keys + i + j;
 601                                 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 602                                                    (const void **)
 603                                                    ((uintptr_t)temp_a),
 604                                                    BULK_LOOKUP_SIZE, pos);
 605                                 /* Validate lookup result */
 606                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 607                                         if ((read_type & READ_FAIL &&
 608                                              pos[j] != -ENOENT) ||
 609                                             (!(read_type & READ_FAIL) &&
 610                                              pos[j] == -ENOENT)) {
 611                                                 printf("lookup failed!"
 612                                                        "%"PRIu32"\n",
 613                                                        keys[i + j]);
 614                                                 return -1;
 615                                         }
 616                         }
 617                         for (j = 0; j < extra_keys; j++)
 618                                 temp_a[j] = keys + i + j;
 619
 620                         rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 621                                            (const void **)
 622                                            ((uintptr_t)temp_a),
 623                                            extra_keys, pos);
 624                         for (j = 0; j < extra_keys; j++)
 625                                 if ((read_type & READ_FAIL &&
 626                                      pos[j] != -ENOENT) ||
 627                                     (!(read_type & READ_FAIL) &&
 628                                      pos[j] == -ENOENT)) {
 629                                         printf("lookup failed! %"PRIu32"\n",
 630                                                keys[i + j]);
 631                                         return -1;
 632                                 }
 633                 } else {
 634                         for (i = 0; i < read_cnt; i++) {
 635                                 ret = rte_hash_lookup
 636                                         (tbl_rwc_test_param.h, keys + i);
 637                                 if (((read_type & READ_FAIL) &&
 638                                      (ret != -ENOENT)) ||
 639                                     (!(read_type & READ_FAIL) &&
 640                                         ret == -ENOENT)) {
 641                                         printf("lookup failed! %"PRIu32"\n",
 642                                                keys[i]);
 643                                         return -1;
 644                                 }
 645                         }
 646                 }
 647                 loop_cnt++;
 648         } while (!writer_done);
 649
 650         cycles = rte_rdtsc_precise() - begin;
 651         rte_atomic64_add(&gread_cycles, cycles);
 652         rte_atomic64_add(&greads, read_cnt*loop_cnt);
 653         return 0;
 654 }
 655
 656 static int
 657 write_keys(uint8_t write_type)
 658 {
 659         uint32_t i;
 660         int ret;
 661         uint32_t key_cnt = 0;
 662         uint32_t *keys;
 663         if (write_type == WRITE_KEY_SHIFT) {
 664                 key_cnt = tbl_rwc_test_param.count_keys_ks;
 665                 keys = tbl_rwc_test_param.keys_ks;
 666         } else if (write_type == WRITE_NO_KEY_SHIFT) {
 667                 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
 668                 keys = tbl_rwc_test_param.keys_no_ks;
 669         } else if (write_type == WRITE_EXT_BKT) {
 670                 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
 671                 keys = tbl_rwc_test_param.keys_ext_bkt;
 672         }
 673         for (i = 0; i < key_cnt; i++) {
 674                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
 675                 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
 676                         printf("writer failed %"PRIu32"\n", i);
 677                         return -1;
 678                 }
 679         }
 680         return 0;
 681 }
 682
 683 static int
 684 test_rwc_multi_writer(__attribute__((unused)) void *arg)
 685 {
 686         uint32_t i, offset;
 687         uint32_t pos_core = (uint32_t)((uintptr_t)arg);
 688         offset = pos_core * tbl_rwc_test_param.single_insert;
 689         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
 690                 rte_hash_add_key(tbl_rwc_test_param.h,
 691                                  tbl_rwc_test_param.keys_ks + i);
 692         return 0;
 693 }
 694
 695 /*
 696  * Test lookup perf:
 697  * Reader(s) lookup keys present in the table.
 698  */
 699 static int
 700 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 701                                 int htm, int ext_bkt)
 702 {
 703         unsigned int n, m;
 704         uint64_t i;
 705         int use_jhash = 0;
 706         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 707         uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
 708
 709         rte_atomic64_init(&greads);
 710         rte_atomic64_init(&gread_cycles);
 711
 712         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 713                 goto err;
 714         printf("\nTest: Hash add - no key-shifts, read - hit\n");
 715         for (m = 0; m < 2; m++) {
 716                 if (m == 1) {
 717                         printf("\n** With bulk-lookup **\n");
 718                         read_type |= BULK_LOOKUP;
 719                 }
 720                 for (n = 0; n < NUM_TEST; n++) {
 721                         unsigned int tot_lcore = rte_lcore_count();
 722                         if (tot_lcore < rwc_core_cnt[n] + 1)
 723                                 goto finish;
 724
 725                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 726
 727                         rte_atomic64_clear(&greads);
 728                         rte_atomic64_clear(&gread_cycles);
 729
 730                         rte_hash_reset(tbl_rwc_test_param.h);
 731                         writer_done = 0;
 732                         if (write_keys(write_type) < 0)
 733                                 goto err;
 734                         writer_done = 1;
 735                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 736                                 rte_eal_remote_launch(test_rwc_reader,
 737                                                 (void *)(uintptr_t)read_type,
 738                                                         enabled_core_ids[i]);
 739
 740                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 741                                 if (rte_eal_wait_lcore(i) < 0)
 742                                         goto err;
 743
 744                         unsigned long long cycles_per_lookup =
 745                                 rte_atomic64_read(&gread_cycles) /
 746                                 rte_atomic64_read(&greads);
 747                         rwc_perf_results->w_no_ks_r_hit[m][n]
 748                                                 = cycles_per_lookup;
 749                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 750                 }
 751         }
 752
 753 finish:
 754         rte_hash_free(tbl_rwc_test_param.h);
 755         return 0;
 756
 757 err:
 758         rte_eal_mp_wait_lcore();
 759         rte_hash_free(tbl_rwc_test_param.h);
 760         return -1;
 761 }
 762
 763 /*
 764  * Test lookup perf:
 765  * Reader(s) lookup keys absent in the table while
 766  * 'Main' thread adds with no key-shifts.
 767  */
 768 static int
 769 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 770                                 int htm, int ext_bkt)
 771 {
 772         unsigned int n, m;
 773         uint64_t i;
 774         int use_jhash = 0;
 775         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 776         uint8_t read_type = READ_FAIL;
 777         int ret;
 778
 779         rte_atomic64_init(&greads);
 780         rte_atomic64_init(&gread_cycles);
 781
 782         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 783                 goto err;
 784         printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
 785         for (m = 0; m < 2; m++) {
 786                 if (m == 1) {
 787                         printf("\n** With bulk-lookup **\n");
 788                         read_type |= BULK_LOOKUP;
 789                 }
 790                 for (n = 0; n < NUM_TEST; n++) {
 791                         unsigned int tot_lcore = rte_lcore_count();
 792                         if (tot_lcore < rwc_core_cnt[n] + 1)
 793                                 goto finish;
 794
 795                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 796
 797                         rte_atomic64_clear(&greads);
 798                         rte_atomic64_clear(&gread_cycles);
 799
 800                         rte_hash_reset(tbl_rwc_test_param.h);
 801                         writer_done = 0;
 802
 803                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 804                                 rte_eal_remote_launch(test_rwc_reader,
 805                                                 (void *)(uintptr_t)read_type,
 806                                                         enabled_core_ids[i]);
 807                         ret = write_keys(write_type);
 808                         writer_done = 1;
 809
 810                         if (ret < 0)
 811                                 goto err;
 812                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 813                                 if (rte_eal_wait_lcore(i) < 0)
 814                                         goto err;
 815
 816                         unsigned long long cycles_per_lookup =
 817                                 rte_atomic64_read(&gread_cycles) /
 818                                 rte_atomic64_read(&greads);
 819                         rwc_perf_results->w_no_ks_r_miss[m][n]
 820                                                 = cycles_per_lookup;
 821                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 822                 }
 823         }
 824
 825 finish:
 826         rte_hash_free(tbl_rwc_test_param.h);
 827         return 0;
 828
 829 err:
 830         rte_eal_mp_wait_lcore();
 831         rte_hash_free(tbl_rwc_test_param.h);
 832         return -1;
 833 }
 834
 835 /*
 836  * Test lookup perf:
 837  * Reader(s) lookup keys present in the table and not likely to be on the
 838  * shift path  while 'Main' thread adds keys causing key-shifts.
 839  */
 840 static int
 841 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 842                                     int rwc_lf, int htm, int ext_bkt)
 843 {
 844         unsigned int n, m;
 845         uint64_t i;
 846         int use_jhash = 0;
 847         int ret;
 848         uint8_t write_type;
 849         uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
 850
 851         rte_atomic64_init(&greads);
 852         rte_atomic64_init(&gread_cycles);
 853
 854         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 855                 goto err;
 856         printf("\nTest: Hash add - key shift, Hash lookup - hit"
 857                " (non-shift-path)\n");
 858         for (m = 0; m < 2; m++) {
 859                 if (m == 1) {
 860                         printf("\n** With bulk-lookup **\n");
 861                         read_type |= BULK_LOOKUP;
 862                 }
 863                 for (n = 0; n < NUM_TEST; n++) {
 864                         unsigned int tot_lcore = rte_lcore_count();
 865                         if (tot_lcore < rwc_core_cnt[n] + 1)
 866                                 goto finish;
 867
 868                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 869
 870                         rte_atomic64_clear(&greads);
 871                         rte_atomic64_clear(&gread_cycles);
 872
 873                         rte_hash_reset(tbl_rwc_test_param.h);
 874                         writer_done = 0;
 875                         write_type = WRITE_NO_KEY_SHIFT;
 876                         if (write_keys(write_type) < 0)
 877                                 goto err;
 878                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 879                                 rte_eal_remote_launch(test_rwc_reader,
 880                                                 (void *)(uintptr_t)read_type,
 881                                                         enabled_core_ids[i]);
 882                         write_type = WRITE_KEY_SHIFT;
 883                         ret = write_keys(write_type);
 884                         writer_done = 1;
 885
 886                         if (ret < 0)
 887                                 goto err;
 888                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 889                                 if (rte_eal_wait_lcore(i) < 0)
 890                                         goto err;
 891
 892                         unsigned long long cycles_per_lookup =
 893                                 rte_atomic64_read(&gread_cycles) /
 894                                 rte_atomic64_read(&greads);
 895                         rwc_perf_results->w_ks_r_hit_nsp[m][n]
 896                                                 = cycles_per_lookup;
 897                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 898                 }
 899         }
 900
 901 finish:
 902         rte_hash_free(tbl_rwc_test_param.h);
 903         return 0;
 904
 905 err:
 906         rte_eal_mp_wait_lcore();
 907         rte_hash_free(tbl_rwc_test_param.h);
 908         return -1;
 909 }
 910
 911 /*
 912  * Test lookup perf:
 913  * Reader(s) lookup keys present in the table and likely on the shift-path while
 914  * 'Main' thread adds keys causing key-shifts.
 915  */
 916 static int
 917 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 918                                 int htm, int ext_bkt)
 919 {
 920         unsigned int n, m;
 921         uint64_t i;
 922         int use_jhash = 0;
 923         int ret;
 924         uint8_t write_type;
 925         uint8_t read_type = READ_PASS_SHIFT_PATH;
 926
 927         rte_atomic64_init(&greads);
 928         rte_atomic64_init(&gread_cycles);
 929
 930         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 931                 goto err;
 932         printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
 933                "\n");
 934
 935         for (m = 0; m < 2; m++) {
 936                 if (m == 1) {
 937                         printf("\n** With bulk-lookup **\n");
 938                         read_type |= BULK_LOOKUP;
 939                 }
 940                 for (n = 0; n < NUM_TEST; n++) {
 941                         unsigned int tot_lcore = rte_lcore_count();
 942                         if (tot_lcore < rwc_core_cnt[n] + 1)
 943                                 goto finish;
 944
 945                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 946                         rte_atomic64_clear(&greads);
 947                         rte_atomic64_clear(&gread_cycles);
 948
 949                         rte_hash_reset(tbl_rwc_test_param.h);
 950                         writer_done = 0;
 951                         write_type = WRITE_NO_KEY_SHIFT;
 952                         if (write_keys(write_type) < 0)
 953                                 goto err;
 954                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 955                                 rte_eal_remote_launch(test_rwc_reader,
 956                                                 (void *)(uintptr_t)read_type,
 957                                                 enabled_core_ids[i]);
 958                         write_type = WRITE_KEY_SHIFT;
 959                         ret = write_keys(write_type);
 960                         writer_done = 1;
 961
 962                         if (ret < 0)
 963                                 goto err;
 964                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 965                                 if (rte_eal_wait_lcore(i) < 0)
 966                                         goto err;
 967
 968                         unsigned long long cycles_per_lookup =
 969                                 rte_atomic64_read(&gread_cycles) /
 970                                 rte_atomic64_read(&greads);
 971                         rwc_perf_results->w_ks_r_hit_sp[m][n]
 972                                                 = cycles_per_lookup;
 973                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 974                 }
 975         }
 976
 977 finish:
 978         rte_hash_free(tbl_rwc_test_param.h);
 979         return 0;
 980
 981 err:
 982         rte_eal_mp_wait_lcore();
 983         rte_hash_free(tbl_rwc_test_param.h);
 984         return -1;
 985 }
 986
 987 /*
 988  * Test lookup perf:
 989  * Reader(s) lookup keys absent in the table while
 990  * 'Main' thread adds keys causing key-shifts.
 991  */
 992 static int
 993 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 994                              htm, int ext_bkt)
 995 {
 996         unsigned int n, m;
 997         uint64_t i;
 998         int use_jhash = 0;
 999         int ret;
1000         uint8_t write_type;
1001         uint8_t read_type = READ_FAIL;
1002
1003         rte_atomic64_init(&greads);
1004         rte_atomic64_init(&gread_cycles);
1005
1006         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1007                 goto err;
1008         printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1009         for (m = 0; m < 2; m++) {
1010                 if (m == 1) {
1011                         printf("\n** With bulk-lookup **\n");
1012                         read_type |= BULK_LOOKUP;
1013                 }
1014                 for (n = 0; n < NUM_TEST; n++) {
1015                         unsigned int tot_lcore = rte_lcore_count();
1016                         if (tot_lcore < rwc_core_cnt[n] + 1)
1017                                 goto finish;
1018
1019                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1020
1021                         rte_atomic64_clear(&greads);
1022                         rte_atomic64_clear(&gread_cycles);
1023
1024                         rte_hash_reset(tbl_rwc_test_param.h);
1025                         writer_done = 0;
1026                         write_type = WRITE_NO_KEY_SHIFT;
1027                         if (write_keys(write_type) < 0)
1028                                 goto err;
1029                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1030                                 rte_eal_remote_launch(test_rwc_reader,
1031                                                 (void *)(uintptr_t)read_type,
1032                                                         enabled_core_ids[i]);
1033                         write_type = WRITE_KEY_SHIFT;
1034                         ret = write_keys(write_type);
1035                         writer_done = 1;
1036
1037                         if (ret < 0)
1038                                 goto err;
1039                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1040                                 if (rte_eal_wait_lcore(i) < 0)
1041                                         goto err;
1042
1043                         unsigned long long cycles_per_lookup =
1044                                 rte_atomic64_read(&gread_cycles) /
1045                                 rte_atomic64_read(&greads);
1046                         rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1047                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1048                 }
1049         }
1050
1051 finish:
1052         rte_hash_free(tbl_rwc_test_param.h);
1053         return 0;
1054
1055 err:
1056         rte_eal_mp_wait_lcore();
1057         rte_hash_free(tbl_rwc_test_param.h);
1058         return -1;
1059 }
1060
1061 /*
1062  * Test lookup perf for multi-writer:
1063  * Reader(s) lookup keys present in the table and likely on the shift-path while
1064  * Writers add keys causing key-shiftsi.
1065  * Writers are running in parallel, on different data plane cores.
1066  */
1067 static int
1068 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1069                            int htm, int ext_bkt)
1070 {
1071         unsigned int n, m, k;
1072         uint64_t i;
1073         int use_jhash = 0;
1074         uint8_t write_type;
1075         uint8_t read_type = READ_PASS_SHIFT_PATH;
1076
1077         rte_atomic64_init(&greads);
1078         rte_atomic64_init(&gread_cycles);
1079
1080         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1081                 goto err;
1082         printf("\nTest: Multi-add-lookup\n");
1083         uint8_t pos_core;
1084         for (m = 1; m < NUM_TEST; m++) {
1085                 /* Calculate keys added by each writer */
1086                 tbl_rwc_test_param.single_insert =
1087                         tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1088                 for (k = 0; k < 2; k++) {
1089                         if (k == 1) {
1090                                 printf("\n** With bulk-lookup **\n");
1091                                 read_type |= BULK_LOOKUP;
1092                         }
1093                         for (n = 0; n < NUM_TEST; n++) {
1094                                 unsigned int tot_lcore  = rte_lcore_count();
1095                                 if (tot_lcore < (rwc_core_cnt[n] +
1096                                      rwc_core_cnt[m] + 1))
1097                                         goto finish;
1098
1099                                 printf("\nNumber of writers: %u",
1100                                        rwc_core_cnt[m]);
1101                                 printf("\nNumber of readers: %u\n",
1102                                        rwc_core_cnt[n]);
1103
1104                                 rte_atomic64_clear(&greads);
1105                                 rte_atomic64_clear(&gread_cycles);
1106
1107                                 rte_hash_reset(tbl_rwc_test_param.h);
1108                                 writer_done = 0;
1109                                 write_type = WRITE_NO_KEY_SHIFT;
1110                                 if (write_keys(write_type) < 0)
1111                                         goto err;
1112
1113                                 /* Launch reader(s) */
1114                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1115                                         rte_eal_remote_launch(test_rwc_reader,
1116                                                 (void *)(uintptr_t)read_type,
1117                                                 enabled_core_ids[i]);
1118                                 write_type = WRITE_KEY_SHIFT;
1119                                 pos_core = 0;
1120
1121                                 /* Launch writers */
1122                                 for (; i <= rwc_core_cnt[m]
1123                                      + rwc_core_cnt[n]; i++) {
1124                                         rte_eal_remote_launch
1125                                                 (test_rwc_multi_writer,
1126                                                 (void *)(uintptr_t)pos_core,
1127                                                 enabled_core_ids[i]);
1128                                         pos_core++;
1129                                 }
1130
1131                                 /* Wait for writers to complete */
1132                                 for (i = rwc_core_cnt[n] + 1;
1133                                      i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1134                                      i++)
1135                                         rte_eal_wait_lcore(i);
1136
1137                                 writer_done = 1;
1138
1139                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1140                                         if (rte_eal_wait_lcore(i) < 0)
1141                                                 goto err;
1142
1143                                 unsigned long long cycles_per_lookup =
1144                                         rte_atomic64_read(&gread_cycles)
1145                                         / rte_atomic64_read(&greads);
1146                                 rwc_perf_results->multi_rw[m][k][n]
1147                                         = cycles_per_lookup;
1148                                 printf("Cycles per lookup: %llu\n",
1149                                        cycles_per_lookup);
1150                         }
1151                 }
1152         }
1153
1154 finish:
1155         rte_hash_free(tbl_rwc_test_param.h);
1156         return 0;
1157
1158 err:
1159         rte_eal_mp_wait_lcore();
1160         rte_hash_free(tbl_rwc_test_param.h);
1161         return -1;
1162 }
1163
1164 /*
1165  * Test lookup perf:
1166  * Reader(s) lookup keys present in the extendable bkt.
1167  */
1168 static int
1169 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1170                                 int rwc_lf, int htm, int ext_bkt)
1171 {
1172         unsigned int n, m;
1173         uint64_t i;
1174         int use_jhash = 0;
1175         uint8_t write_type;
1176         uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1177
1178         rte_atomic64_init(&greads);
1179         rte_atomic64_init(&gread_cycles);
1180
1181         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1182                 goto err;
1183         printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1184         for (m = 0; m < 2; m++) {
1185                 if (m == 1) {
1186                         printf("\n** With bulk-lookup **\n");
1187                         read_type |= BULK_LOOKUP;
1188                 }
1189                 for (n = 0; n < NUM_TEST; n++) {
1190                         unsigned int tot_lcore = rte_lcore_count();
1191                         if (tot_lcore < rwc_core_cnt[n] + 1)
1192                                 goto finish;
1193
1194                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1195
1196                         rte_atomic64_clear(&greads);
1197                         rte_atomic64_clear(&gread_cycles);
1198
1199                         rte_hash_reset(tbl_rwc_test_param.h);
1200                         write_type = WRITE_NO_KEY_SHIFT;
1201                         if (write_keys(write_type) < 0)
1202                                 goto err;
1203                         write_type = WRITE_KEY_SHIFT;
1204                         if (write_keys(write_type) < 0)
1205                                 goto err;
1206                         writer_done = 0;
1207                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1208                                 rte_eal_remote_launch(test_rwc_reader,
1209                                                 (void *)(uintptr_t)read_type,
1210                                                         enabled_core_ids[i]);
1211                         for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1212                              i++) {
1213                                 if (rte_hash_del_key(tbl_rwc_test_param.h,
1214                                         tbl_rwc_test_param.keys_ks_extbkt + i)
1215                                                         < 0) {
1216                                         printf("Delete Failed: %u\n",
1217                                         tbl_rwc_test_param.keys_ks_extbkt[i]);
1218                                         goto err;
1219                                 }
1220                         }
1221                         writer_done = 1;
1222
1223                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1224                                 if (rte_eal_wait_lcore(i) < 0)
1225                                         goto err;
1226
1227                         unsigned long long cycles_per_lookup =
1228                                 rte_atomic64_read(&gread_cycles) /
1229                                 rte_atomic64_read(&greads);
1230                         rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1231                                                 = cycles_per_lookup;
1232                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1233                 }
1234         }
1235
1236 finish:
1237         rte_hash_free(tbl_rwc_test_param.h);
1238         return 0;
1239
1240 err:
1241         rte_eal_mp_wait_lcore();
1242         rte_hash_free(tbl_rwc_test_param.h);
1243         return -1;
1244 }
1245
1246 static int
1247 test_hash_readwrite_lf_main(void)
1248 {
1249         /*
1250          * Variables used to choose different tests.
1251          * rwc_lf indicates if read-write concurrency lock-free support is
1252          * enabled.
1253          * htm indicates if Hardware transactional memory support is enabled.
1254          */
1255         int rwc_lf = 0;
1256         int htm;
1257         int use_jhash = 0;
1258         int ext_bkt = 0;
1259         if (rte_lcore_count() == 1) {
1260                 printf("More than one lcore is required "
1261                         "to do read write lock-free concurrency test\n");
1262                 return -1;
1263         }
1264
1265         setlocale(LC_NUMERIC, "");
1266
1267         if (rte_tm_supported())
1268                 htm = 1;
1269         else
1270                 htm = 0;
1271
1272         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1273                 return -1;
1274         if (generate_keys() != 0)
1275                 return -1;
1276         if (get_enabled_cores_list() != 0)
1277                 return -1;
1278
1279         if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1280                 rwc_lf = 1;
1281                 ext_bkt = 1;
1282                 printf("Test lookup with read-write concurrency lock free support"
1283                        " enabled\n");
1284                 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1285                                                         htm, ext_bkt) < 0)
1286                         return -1;
1287                 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1288                                                         htm, ext_bkt) < 0)
1289                         return -1;
1290                 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1291                                                         htm, ext_bkt) < 0)
1292                         return -1;
1293                 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1294                                                         htm, ext_bkt) < 0)
1295                         return -1;
1296                 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1297                                                  ext_bkt) < 0)
1298                         return -1;
1299                 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1300                                                ext_bkt) < 0)
1301                         return -1;
1302                 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1303                                                         htm, ext_bkt) < 0)
1304                         return -1;
1305         }
1306         printf("\nTest lookup with read-write concurrency lock free support"
1307                " disabled\n");
1308         rwc_lf = 0;
1309         if (!htm) {
1310                 printf("With HTM Disabled\n");
1311                 if (!RUN_WITH_HTM_DISABLED) {
1312                         printf("Enable RUN_WITH_HTM_DISABLED to test with"
1313                                " lock-free disabled");
1314                         goto results;
1315                 }
1316         } else
1317                 printf("With HTM Enabled\n");
1318         if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1319                                            ext_bkt) < 0)
1320                 return -1;
1321         if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1322                                                 ext_bkt) < 0)
1323                 return -1;
1324         if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1325                                                 htm, ext_bkt) < 0)
1326                 return -1;
1327         if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1328                                                 ext_bkt) < 0)
1329                 return -1;
1330         if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1331                                          ext_bkt) < 0)
1332                 return -1;
1333         if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1334                                                         ext_bkt) < 0)
1335                 return -1;
1336         if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1337                                                 htm, ext_bkt) < 0)
1338                 return -1;
1339 results:
1340         printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1341         int i, j, k;
1342         for (j = 0; j < 2; j++) {
1343                 if (j == 1)
1344                         printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1345                                "**********#######\n\n");
1346                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1347                         "\t\t\t\t_________________\n");
1348                 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1349                        "\t\t\tCycles per lookup\n");
1350                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1351                        "\t\t\t_________________\n");
1352                 for (i = 0; i < NUM_TEST; i++) {
1353                         printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1354                         printf("Enabled\t\t");
1355                         printf("N/A\t\t");
1356                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1357                                 "%u\n\t\t\t\t\t\t\t\t",
1358                                 rwc_lf_results.w_no_ks_r_hit[j][i]);
1359                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1360                                 "%u\n\t\t\t\t\t\t\t\t",
1361                                 rwc_lf_results.w_no_ks_r_miss[j][i]);
1362                         printf("Hash add - key-shifts, lookup - hit"
1363                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1364                                rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1365                         printf("Hash add - key-shifts, lookup - hit "
1366                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1367                                rwc_lf_results.w_ks_r_hit_sp[j][i]);
1368                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1369                                 "%u\n\t\t\t\t\t\t\t\t",
1370                                 rwc_lf_results.w_ks_r_miss[j][i]);
1371                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1372                                 "%u\n\n\t\t\t\t",
1373                                 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1374
1375                         printf("Disabled\t");
1376                         if (htm)
1377                                 printf("Enabled\t\t");
1378                         else
1379                                 printf("Disabled\t");
1380                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1381                                 "%u\n\t\t\t\t\t\t\t\t",
1382                                 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1383                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1384                                 "%u\n\t\t\t\t\t\t\t\t",
1385                                 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1386                         printf("Hash add - key-shifts, lookup - hit "
1387                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1388                                rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1389                         printf("Hash add - key-shifts, lookup - hit "
1390                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1391                                rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1392                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1393                                "%u\n\t\t\t\t\t\t\t\t",
1394                                rwc_non_lf_results.w_ks_r_miss[j][i]);
1395                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1396                                 "%u\n",
1397                                 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1398
1399                         printf("_______\t\t_______\t\t_________\t___\t\t"
1400                                "_________\t\t\t\t\t\t_________________\n");
1401                 }
1402
1403                 for (i = 1; i < NUM_TEST; i++) {
1404                         for (k = 0; k < NUM_TEST; k++) {
1405                                 printf("%u", rwc_core_cnt[i]);
1406                                 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1407                                 printf("Enabled\t\t");
1408                                 printf("N/A\t\t");
1409                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1410                                        "\t\t",
1411                                        rwc_lf_results.multi_rw[i][j][k]);
1412                                 printf("Disabled\t");
1413                                 if (htm)
1414                                         printf("Enabled\t\t");
1415                                 else
1416                                         printf("Disabled\t");
1417                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1418                                        rwc_non_lf_results.multi_rw[i][j][k]);
1419
1420                                 printf("_______\t\t_______\t\t_________\t___"
1421                                        "\t\t_________\t\t\t\t\t\t"
1422                                        "_________________\n");
1423                         }
1424                 }
1425         }
1426         rte_free(tbl_rwc_test_param.keys);
1427         rte_free(tbl_rwc_test_param.keys_no_ks);
1428         rte_free(tbl_rwc_test_param.keys_ks);
1429         rte_free(tbl_rwc_test_param.keys_absent);
1430         rte_free(tbl_rwc_test_param.keys_shift_path);
1431         rte_free(scanned_bkts);
1432         return 0;
1433 }
1434
1435 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);