app/test/test_hash_readwrite_lf_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Arm Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
  20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
  21 #endif
  22
  23 #define BULK_LOOKUP_SIZE 32
  24
  25 #define RUN_WITH_HTM_DISABLED 0
  26
  27 #if (RUN_WITH_HTM_DISABLED)
  28
  29 #define TOTAL_ENTRY (5*1024)
  30 #define TOTAL_INSERT (5*1024)
  31
  32 #else
  33
  34 #define TOTAL_ENTRY (4*1024*1024)
  35 #define TOTAL_INSERT (4*1024*1024)
  36
  37 #endif
  38
  39 #define READ_FAIL 1
  40 #define READ_PASS_NO_KEY_SHIFTS 2
  41 #define READ_PASS_SHIFT_PATH 4
  42 #define READ_PASS_NON_SHIFT_PATH 8
  43 #define BULK_LOOKUP 16
  44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
  45
  46 #define WRITE_NO_KEY_SHIFT 0
  47 #define WRITE_KEY_SHIFT 1
  48 #define WRITE_EXT_BKT 2
  49
  50 #define NUM_TEST 3
  51
  52 #define QSBR_REPORTING_INTERVAL 1024
  53
  54 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
  55
  56 struct rwc_perf {
  57         uint32_t w_no_ks_r_hit[2][NUM_TEST];
  58         uint32_t w_no_ks_r_miss[2][NUM_TEST];
  59         uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
  60         uint32_t w_ks_r_hit_sp[2][NUM_TEST];
  61         uint32_t w_ks_r_miss[2][NUM_TEST];
  62         uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
  63         uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
  64         uint32_t writer_add_del[NUM_TEST];
  65 };
  66
  67 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
  68
  69 static struct {
  70         uint32_t *keys;
  71         uint32_t *keys_no_ks;
  72         uint32_t *keys_ks;
  73         uint32_t *keys_absent;
  74         uint32_t *keys_shift_path;
  75         uint32_t *keys_non_shift_path;
  76         uint32_t *keys_ext_bkt;
  77         uint32_t *keys_ks_extbkt;
  78         uint32_t count_keys_no_ks;
  79         uint32_t count_keys_ks;
  80         uint32_t count_keys_absent;
  81         uint32_t count_keys_shift_path;
  82         uint32_t count_keys_non_shift_path;
  83         uint32_t count_keys_extbkt;
  84         uint32_t count_keys_ks_extbkt;
  85         uint32_t single_insert;
  86         struct rte_hash *h;
  87 } tbl_rwc_test_param;
  88
  89 static uint64_t gread_cycles;
  90 static uint64_t greads;
  91 static uint64_t gwrite_cycles;
  92 static uint64_t gwrites;
  93
  94 static volatile uint8_t writer_done;
  95
  96 static uint16_t enabled_core_ids[RTE_MAX_LCORE];
  97
  98 static uint8_t *scanned_bkts;
  99
 100 static inline uint16_t
 101 get_short_sig(const hash_sig_t hash)
 102 {
 103         return hash >> 16;
 104 }
 105
 106 static inline uint32_t
 107 get_prim_bucket_index(__rte_unused const struct rte_hash *h,
 108                       const hash_sig_t hash)
 109 {
 110         uint32_t num_buckets;
 111         uint32_t bucket_bitmask;
 112         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 113         bucket_bitmask = num_buckets - 1;
 114         return hash & bucket_bitmask;
 115 }
 116
 117 static inline uint32_t
 118 get_alt_bucket_index(__rte_unused const struct rte_hash *h,
 119                         uint32_t cur_bkt_idx, uint16_t sig)
 120 {
 121         uint32_t num_buckets;
 122         uint32_t bucket_bitmask;
 123         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 124         bucket_bitmask = num_buckets - 1;
 125         return (cur_bkt_idx ^ sig) & bucket_bitmask;
 126 }
 127
 128
 129 static inline int
 130 get_enabled_cores_list(void)
 131 {
 132         uint32_t i = 0;
 133         uint16_t core_id;
 134         uint32_t max_cores = rte_lcore_count();
 135         RTE_LCORE_FOREACH(core_id) {
 136                 enabled_core_ids[i] = core_id;
 137                 i++;
 138         }
 139
 140         if (i != max_cores) {
 141                 printf("Number of enabled cores in list is different from "
 142                         "number given by rte_lcore_count()\n");
 143                 return -1;
 144         }
 145         return 0;
 146 }
 147
 148 static int
 149 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
 150 {
 151         struct rte_hash *handle;
 152
 153         struct rte_hash_parameters hash_params = {
 154                 .entries = TOTAL_ENTRY,
 155                 .key_len = sizeof(uint32_t),
 156                 .hash_func_init_val = 0,
 157                 .socket_id = rte_socket_id(),
 158         };
 159
 160         if (use_jhash)
 161                 hash_params.hash_func = rte_jhash;
 162         else
 163                 hash_params.hash_func = rte_hash_crc;
 164
 165         if (rwc_lf)
 166                 hash_params.extra_flag =
 167                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
 168                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 169         else if (htm)
 170                 hash_params.extra_flag =
 171                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 172                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 173                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 174         else
 175                 hash_params.extra_flag =
 176                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 177                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 178
 179         if (ext_bkt)
 180                 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
 181
 182         hash_params.name = "tests";
 183
 184         handle = rte_hash_create(&hash_params);
 185         if (handle == NULL) {
 186                 printf("hash creation failed");
 187                 return -1;
 188         }
 189
 190         tbl_rwc_test_param.h = handle;
 191         return 0;
 192 }
 193
 194 static inline int
 195 check_bucket(uint32_t bkt_idx, uint32_t key)
 196 {
 197         uint32_t iter;
 198         uint32_t prev_iter;
 199         uint32_t diff;
 200         uint32_t count = 0;
 201         const void *next_key;
 202         void *next_data;
 203
 204         /* Temporary bucket to hold the keys */
 205         uint32_t keys_in_bkt[8];
 206
 207         iter = bkt_idx * 8;
 208         prev_iter = iter;
 209         while (rte_hash_iterate(tbl_rwc_test_param.h,
 210                         &next_key, &next_data, &iter) >= 0) {
 211
 212                 /* Check for duplicate entries */
 213                 if (*(const uint32_t *)next_key == key)
 214                         return 1;
 215
 216                 /* Identify if there is any free entry in the bucket */
 217                 diff = iter - prev_iter;
 218                 if (diff > 1)
 219                         break;
 220
 221                 prev_iter = iter;
 222                 keys_in_bkt[count] = *(const uint32_t *)next_key;
 223                 count++;
 224
 225                 /* All entries in the bucket are occupied */
 226                 if (count == 8) {
 227
 228                         /*
 229                          * Check if bucket was not scanned before, to avoid
 230                          * duplicate keys.
 231                          */
 232                         if (scanned_bkts[bkt_idx] == 0) {
 233                                 /*
 234                                  * Since this bucket (pointed to by bkt_idx) is
 235                                  * full, it is likely that key(s) in this
 236                                  * bucket will be on the shift path, when
 237                                  * collision occurs. Thus, add it to
 238                                  * keys_shift_path.
 239                                  */
 240                                 memcpy(tbl_rwc_test_param.keys_shift_path +
 241                                         tbl_rwc_test_param.count_keys_shift_path
 242                                         , keys_in_bkt, 32);
 243                                 tbl_rwc_test_param.count_keys_shift_path += 8;
 244                                 scanned_bkts[bkt_idx] = 1;
 245                         }
 246                         return -1;
 247                 }
 248         }
 249         return 0;
 250 }
 251
 252 static int
 253 generate_keys(void)
 254 {
 255         uint32_t *keys = NULL;
 256         uint32_t *keys_no_ks = NULL;
 257         uint32_t *keys_ks = NULL;
 258         uint32_t *keys_absent = NULL;
 259         uint32_t *keys_non_shift_path = NULL;
 260         uint32_t *keys_ext_bkt = NULL;
 261         uint32_t *keys_ks_extbkt = NULL;
 262         uint32_t *found = NULL;
 263         uint32_t count_keys_no_ks = 0;
 264         uint32_t count_keys_ks = 0;
 265         uint32_t count_keys_extbkt = 0;
 266         uint32_t i;
 267
 268         if (init_params(0, 0, 0, 0) != 0)
 269                 return -1;
 270
 271         /*
 272          * keys will consist of a) keys whose addition to the hash table
 273          * will result in shifting of the existing keys to their alternate
 274          * locations b) keys whose addition to the hash table will not result
 275          * in shifting of the existing keys.
 276          */
 277         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 278         if (keys == NULL) {
 279                 printf("RTE_MALLOC failed\n");
 280                 goto err;
 281         }
 282
 283         /*
 284          * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
 285          * will NOT result in shifting of the existing keys to their alternate
 286          * locations. Roughly around 900K keys.
 287          */
 288         keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 289         if (keys_no_ks == NULL) {
 290                 printf("RTE_MALLOC failed\n");
 291                 goto err;
 292         }
 293
 294         /*
 295          * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
 296          * result in shifting of the existing keys to their alternate locations.
 297          * Roughly around 146K keys. There might be repeating keys. More code is
 298          * required to filter out these keys which will complicate the test case
 299          */
 300         keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 301         if (keys_ks == NULL) {
 302                 printf("RTE_MALLOC failed\n");
 303                 goto err;
 304         }
 305
 306         /* Used to identify keys not inserted in the hash table */
 307         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 308         if (found == NULL) {
 309                 printf("RTE_MALLOC failed\n");
 310                 goto err;
 311         }
 312
 313         /*
 314          * This consist of keys not inserted to the hash table.
 315          * Used to test perf of lookup on keys that do not exist in the table.
 316          */
 317         keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 318         if (keys_absent == NULL) {
 319                 printf("RTE_MALLOC failed\n");
 320                 goto err;
 321         }
 322
 323         /*
 324          * This consist of keys which are likely to be on the shift
 325          * path (i.e. being moved to alternate location), when collision occurs
 326          * on addition of a key to an already full primary bucket.
 327          * Used to test perf of lookup on keys that are on the shift path.
 328          */
 329         tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
 330                                                         TOTAL_INSERT, 0);
 331         if (tbl_rwc_test_param.keys_shift_path == NULL) {
 332                 printf("RTE_MALLOC failed\n");
 333                 goto err;
 334         }
 335
 336         /*
 337          * This consist of keys which are never on the shift
 338          * path (i.e. being moved to alternate location), when collision occurs
 339          * on addition of a key to an already full primary bucket.
 340          * Used to test perf of lookup on keys that are not on the shift path.
 341          */
 342         keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
 343                                          0);
 344         if (keys_non_shift_path == NULL) {
 345                 printf("RTE_MALLOC failed\n");
 346                 goto err;
 347         }
 348
 349         /*
 350          * This consist of keys which will be stored in extended buckets
 351          */
 352         keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 353         if (keys_ext_bkt == NULL) {
 354                 printf("RTE_MALLOC failed\n");
 355                 goto err;
 356         }
 357
 358         /*
 359          * This consist of keys which when deleted causes shifting of keys
 360          * in extended buckets to respective secondary buckets
 361          */
 362         keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 363         if (keys_ks_extbkt == NULL) {
 364                 printf("RTE_MALLOC failed\n");
 365                 goto err;
 366         }
 367
 368         hash_sig_t sig;
 369         uint32_t prim_bucket_idx;
 370         uint32_t sec_bucket_idx;
 371         uint16_t short_sig;
 372         uint32_t num_buckets;
 373         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 374         int ret;
 375
 376         /*
 377          * Used to mark bkts in which at least one key was shifted to its
 378          * alternate location
 379          */
 380         scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
 381         if (scanned_bkts == NULL) {
 382                 printf("RTE_MALLOC failed\n");
 383                 goto err;
 384         }
 385
 386         tbl_rwc_test_param.keys = keys;
 387         tbl_rwc_test_param.keys_no_ks = keys_no_ks;
 388         tbl_rwc_test_param.keys_ks = keys_ks;
 389         tbl_rwc_test_param.keys_absent = keys_absent;
 390         tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
 391         tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
 392         tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
 393         /* Generate keys by adding previous two keys, neglect overflow */
 394         printf("Generating keys...\n");
 395         keys[0] = 0;
 396         keys[1] = 1;
 397         for (i = 2; i < TOTAL_INSERT; i++)
 398                 keys[i] = keys[i-1] + keys[i-2];
 399
 400         /* Segregate keys into keys_no_ks and keys_ks */
 401         for (i = 0; i < TOTAL_INSERT; i++) {
 402                 /* Check if primary bucket has space.*/
 403                 sig = rte_hash_hash(tbl_rwc_test_param.h,
 404                                         tbl_rwc_test_param.keys+i);
 405                 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
 406                                                         sig);
 407                 ret = check_bucket(prim_bucket_idx, keys[i]);
 408                 if (ret < 0) {
 409                         /*
 410                          * Primary bucket is full, this key will result in
 411                          * shifting of the keys to their alternate locations.
 412                          */
 413                         keys_ks[count_keys_ks] = keys[i];
 414                         count_keys_ks++;
 415                 } else if (ret == 0) {
 416                         /*
 417                          * Primary bucket has space, this key will not result in
 418                          * shifting of the keys. Hence, add key to the table.
 419                          */
 420                         ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
 421                                                         keys+i,
 422                                                         (void *)((uintptr_t)i));
 423                         if (ret < 0) {
 424                                 printf("writer failed %"PRIu32"\n", i);
 425                                 break;
 426                         }
 427                         keys_no_ks[count_keys_no_ks] = keys[i];
 428                         count_keys_no_ks++;
 429                 }
 430         }
 431
 432         for (i = 0; i < count_keys_no_ks; i++) {
 433                 /*
 434                  * Identify keys in keys_no_ks with value less than
 435                  * 4M (HTM enabled) OR 5K (HTM disabled)
 436                  */
 437                 if (keys_no_ks[i] < TOTAL_INSERT)
 438                         found[keys_no_ks[i]]++;
 439         }
 440
 441         for (i = 0; i < count_keys_ks; i++) {
 442                 /*
 443                  * Identify keys in keys_ks with value less than
 444                  * 4M (HTM enabled) OR 5K (HTM disabled)
 445                  */
 446                 if (keys_ks[i] < TOTAL_INSERT)
 447                         found[keys_ks[i]]++;
 448         }
 449
 450         uint32_t count_keys_absent = 0;
 451         for (i = 0; i < TOTAL_INSERT; i++) {
 452                 /*
 453                  * Identify missing keys between 0 and
 454                  * 4M (HTM enabled) OR 5K (HTM disabled)
 455                  */
 456                 if (found[i] == 0)
 457                         keys_absent[count_keys_absent++] = i;
 458         }
 459
 460         /* Find keys that will not be on the shift path */
 461         uint32_t iter;
 462         const void *next_key;
 463         void *next_data;
 464         uint32_t count = 0;
 465         for (i = 0; i < num_buckets; i++) {
 466                 /* Check bucket for no keys shifted to alternate locations */
 467                 if (scanned_bkts[i] == 0) {
 468                         iter = i * 8;
 469                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 470                                 &next_key, &next_data, &iter) >= 0) {
 471
 472                                 /* Check if key belongs to the current bucket */
 473                                 if (i >= (iter-1)/8)
 474                                         keys_non_shift_path[count++]
 475                                                 = *(const uint32_t *)next_key;
 476                                 else
 477                                         break;
 478                         }
 479                 }
 480         }
 481
 482         tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
 483         tbl_rwc_test_param.count_keys_ks = count_keys_ks;
 484         tbl_rwc_test_param.count_keys_absent = count_keys_absent;
 485         tbl_rwc_test_param.count_keys_non_shift_path = count;
 486
 487         memset(scanned_bkts, 0, num_buckets);
 488         count = 0;
 489         /* Find keys that will be in extended buckets */
 490         for (i = 0; i < count_keys_ks; i++) {
 491                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
 492                 if (ret < 0) {
 493                         /* Key will be added to ext bkt */
 494                         keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
 495                         /* Sec bkt to be added to keys_ks_extbkt */
 496                         sig = rte_hash_hash(tbl_rwc_test_param.h,
 497                                         tbl_rwc_test_param.keys_ks + i);
 498                         prim_bucket_idx = get_prim_bucket_index(
 499                                                 tbl_rwc_test_param.h, sig);
 500                         short_sig = get_short_sig(sig);
 501                         sec_bucket_idx = get_alt_bucket_index(
 502                                                 tbl_rwc_test_param.h,
 503                                                 prim_bucket_idx, short_sig);
 504                         if (scanned_bkts[sec_bucket_idx] == 0)
 505                                 scanned_bkts[sec_bucket_idx] = 1;
 506                 }
 507         }
 508
 509         /* Find keys that will shift keys in ext bucket*/
 510         for (i = 0; i < num_buckets; i++) {
 511                 if (scanned_bkts[i] == 1) {
 512                         iter = i * 8;
 513                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 514                                 &next_key, &next_data, &iter) >= 0) {
 515                                 /* Check if key belongs to the current bucket */
 516                                 if (i >= (iter-1)/8)
 517                                         keys_ks_extbkt[count++]
 518                                                 = *(const uint32_t *)next_key;
 519                                 else
 520                                         break;
 521                         }
 522                 }
 523         }
 524
 525         tbl_rwc_test_param.count_keys_ks_extbkt = count;
 526         tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
 527
 528         printf("\nCount of keys NOT causing shifting of existing keys to "
 529         "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
 530         printf("\nCount of keys causing shifting of existing keys to alternate "
 531                 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
 532         printf("Count of absent keys that will never be added to the hash "
 533                 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
 534         printf("Count of keys likely to be on the shift path: %d\n\n",
 535                tbl_rwc_test_param.count_keys_shift_path);
 536         printf("Count of keys not likely to be on the shift path: %d\n\n",
 537                tbl_rwc_test_param.count_keys_non_shift_path);
 538         printf("Count of keys in extended buckets: %d\n\n",
 539                tbl_rwc_test_param.count_keys_extbkt);
 540         printf("Count of keys shifting keys in ext buckets: %d\n\n",
 541                tbl_rwc_test_param.count_keys_ks_extbkt);
 542
 543         rte_free(found);
 544         rte_free(scanned_bkts);
 545         rte_hash_free(tbl_rwc_test_param.h);
 546         return 0;
 547
 548 err:
 549         rte_free(keys);
 550         rte_free(keys_no_ks);
 551         rte_free(keys_ks);
 552         rte_free(keys_absent);
 553         rte_free(found);
 554         rte_free(tbl_rwc_test_param.keys_shift_path);
 555         rte_free(keys_non_shift_path);
 556         rte_free(keys_ext_bkt);
 557         rte_free(keys_ks_extbkt);
 558         rte_free(scanned_bkts);
 559         rte_hash_free(tbl_rwc_test_param.h);
 560         return -1;
 561 }
 562
 563 static int
 564 test_rwc_reader(__rte_unused void *arg)
 565 {
 566         uint32_t i, j;
 567         int ret;
 568         uint64_t begin, cycles;
 569         uint32_t loop_cnt = 0;
 570         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 571         uint32_t read_cnt;
 572         uint32_t *keys;
 573         uint32_t extra_keys;
 574         int32_t pos[BULK_LOOKUP_SIZE];
 575         void *temp_a[BULK_LOOKUP_SIZE];
 576
 577         if (read_type & READ_FAIL) {
 578                 keys = tbl_rwc_test_param.keys_absent;
 579                 read_cnt = tbl_rwc_test_param.count_keys_absent;
 580         } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
 581                 keys = tbl_rwc_test_param.keys_no_ks;
 582                 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
 583         } else if (read_type & READ_PASS_SHIFT_PATH) {
 584                 keys = tbl_rwc_test_param.keys_shift_path;
 585                 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
 586         } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
 587                 keys = tbl_rwc_test_param.keys_ext_bkt;
 588                 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
 589         } else {
 590                 keys = tbl_rwc_test_param.keys_non_shift_path;
 591                 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
 592         }
 593
 594         extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
 595
 596         begin = rte_rdtsc_precise();
 597         do {
 598                 if (read_type & BULK_LOOKUP) {
 599                         for (i = 0; i < (read_cnt - extra_keys);
 600                              i += BULK_LOOKUP_SIZE) {
 601                                 /* Array of  pointer to the list of keys */
 602                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 603                                         temp_a[j] = keys + i + j;
 604                                 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 605                                                    (const void **)
 606                                                    ((uintptr_t)temp_a),
 607                                                    BULK_LOOKUP_SIZE, pos);
 608                                 /* Validate lookup result */
 609                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 610                                         if ((read_type & READ_FAIL &&
 611                                              pos[j] != -ENOENT) ||
 612                                             (!(read_type & READ_FAIL) &&
 613                                              pos[j] == -ENOENT)) {
 614                                                 printf("lookup failed!"
 615                                                        "%"PRIu32"\n",
 616                                                        keys[i + j]);
 617                                                 return -1;
 618                                         }
 619                         }
 620                         for (j = 0; j < extra_keys; j++)
 621                                 temp_a[j] = keys + i + j;
 622
 623                         rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 624                                            (const void **)
 625                                            ((uintptr_t)temp_a),
 626                                            extra_keys, pos);
 627                         for (j = 0; j < extra_keys; j++)
 628                                 if ((read_type & READ_FAIL &&
 629                                      pos[j] != -ENOENT) ||
 630                                     (!(read_type & READ_FAIL) &&
 631                                      pos[j] == -ENOENT)) {
 632                                         printf("lookup failed! %"PRIu32"\n",
 633                                                keys[i + j]);
 634                                         return -1;
 635                                 }
 636                 } else {
 637                         for (i = 0; i < read_cnt; i++) {
 638                                 ret = rte_hash_lookup
 639                                         (tbl_rwc_test_param.h, keys + i);
 640                                 if (((read_type & READ_FAIL) &&
 641                                      (ret != -ENOENT)) ||
 642                                     (!(read_type & READ_FAIL) &&
 643                                         ret == -ENOENT)) {
 644                                         printf("lookup failed! %"PRIu32"\n",
 645                                                keys[i]);
 646                                         return -1;
 647                                 }
 648                         }
 649                 }
 650                 loop_cnt++;
 651         } while (!writer_done);
 652
 653         cycles = rte_rdtsc_precise() - begin;
 654         __atomic_fetch_add(&gread_cycles, cycles, __ATOMIC_RELAXED);
 655         __atomic_fetch_add(&greads, read_cnt*loop_cnt, __ATOMIC_RELAXED);
 656         return 0;
 657 }
 658
 659 static int
 660 write_keys(uint8_t write_type)
 661 {
 662         uint32_t i;
 663         int ret;
 664         uint32_t key_cnt = 0;
 665         uint32_t *keys;
 666         if (write_type == WRITE_KEY_SHIFT) {
 667                 key_cnt = tbl_rwc_test_param.count_keys_ks;
 668                 keys = tbl_rwc_test_param.keys_ks;
 669         } else if (write_type == WRITE_NO_KEY_SHIFT) {
 670                 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
 671                 keys = tbl_rwc_test_param.keys_no_ks;
 672         } else if (write_type == WRITE_EXT_BKT) {
 673                 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
 674                 keys = tbl_rwc_test_param.keys_ext_bkt;
 675         }
 676         for (i = 0; i < key_cnt; i++) {
 677                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
 678                 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
 679                         printf("writer failed %"PRIu32"\n", i);
 680                         return -1;
 681                 }
 682         }
 683         return 0;
 684 }
 685
 686 static int
 687 test_rwc_multi_writer(__rte_unused void *arg)
 688 {
 689         uint32_t i, offset;
 690         uint32_t pos_core = (uint32_t)((uintptr_t)arg);
 691         offset = pos_core * tbl_rwc_test_param.single_insert;
 692         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
 693                 rte_hash_add_key(tbl_rwc_test_param.h,
 694                                  tbl_rwc_test_param.keys_ks + i);
 695         return 0;
 696 }
 697
 698 /*
 699  * Test lookup perf:
 700  * Reader(s) lookup keys present in the table.
 701  */
 702 static int
 703 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 704                                 int htm, int ext_bkt)
 705 {
 706         unsigned int n, m;
 707         uint64_t i;
 708         int use_jhash = 0;
 709         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 710         uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
 711
 712         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 713                 goto err;
 714         printf("\nTest: Hash add - no key-shifts, read - hit\n");
 715         for (m = 0; m < 2; m++) {
 716                 if (m == 1) {
 717                         printf("\n** With bulk-lookup **\n");
 718                         read_type |= BULK_LOOKUP;
 719                 }
 720                 for (n = 0; n < NUM_TEST; n++) {
 721                         unsigned int tot_lcore = rte_lcore_count();
 722                         if (tot_lcore < rwc_core_cnt[n] + 1)
 723                                 goto finish;
 724
 725                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 726
 727                         __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
 728                         __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
 729
 730                         rte_hash_reset(tbl_rwc_test_param.h);
 731                         writer_done = 0;
 732                         if (write_keys(write_type) < 0)
 733                                 goto err;
 734                         writer_done = 1;
 735                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 736                                 rte_eal_remote_launch(test_rwc_reader,
 737                                                 (void *)(uintptr_t)read_type,
 738                                                         enabled_core_ids[i]);
 739
 740                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 741                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 742                                         goto err;
 743
 744                         unsigned long long cycles_per_lookup =
 745                                 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
 746                                 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
 747                         rwc_perf_results->w_no_ks_r_hit[m][n]
 748                                                 = cycles_per_lookup;
 749                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 750                 }
 751         }
 752
 753 finish:
 754         rte_hash_free(tbl_rwc_test_param.h);
 755         return 0;
 756
 757 err:
 758         rte_eal_mp_wait_lcore();
 759         rte_hash_free(tbl_rwc_test_param.h);
 760         return -1;
 761 }
 762
 763 /*
 764  * Test lookup perf:
 765  * Reader(s) lookup keys absent in the table while
 766  * 'Main' thread adds with no key-shifts.
 767  */
 768 static int
 769 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 770                                 int htm, int ext_bkt)
 771 {
 772         unsigned int n, m;
 773         uint64_t i;
 774         int use_jhash = 0;
 775         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 776         uint8_t read_type = READ_FAIL;
 777         int ret;
 778
 779         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 780                 goto err;
 781         printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
 782         for (m = 0; m < 2; m++) {
 783                 if (m == 1) {
 784                         printf("\n** With bulk-lookup **\n");
 785                         read_type |= BULK_LOOKUP;
 786                 }
 787                 for (n = 0; n < NUM_TEST; n++) {
 788                         unsigned int tot_lcore = rte_lcore_count();
 789                         if (tot_lcore < rwc_core_cnt[n] + 1)
 790                                 goto finish;
 791
 792                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 793
 794                         __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
 795                         __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
 796
 797                         rte_hash_reset(tbl_rwc_test_param.h);
 798                         writer_done = 0;
 799
 800                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 801                                 rte_eal_remote_launch(test_rwc_reader,
 802                                                 (void *)(uintptr_t)read_type,
 803                                                         enabled_core_ids[i]);
 804                         ret = write_keys(write_type);
 805                         writer_done = 1;
 806
 807                         if (ret < 0)
 808                                 goto err;
 809                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 810                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 811                                         goto err;
 812
 813                         unsigned long long cycles_per_lookup =
 814                                 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
 815                                 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
 816                         rwc_perf_results->w_no_ks_r_miss[m][n]
 817                                                 = cycles_per_lookup;
 818                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 819                 }
 820         }
 821
 822 finish:
 823         rte_hash_free(tbl_rwc_test_param.h);
 824         return 0;
 825
 826 err:
 827         rte_eal_mp_wait_lcore();
 828         rte_hash_free(tbl_rwc_test_param.h);
 829         return -1;
 830 }
 831
 832 /*
 833  * Test lookup perf:
 834  * Reader(s) lookup keys present in the table and not likely to be on the
 835  * shift path  while 'Main' thread adds keys causing key-shifts.
 836  */
 837 static int
 838 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 839                                     int rwc_lf, int htm, int ext_bkt)
 840 {
 841         unsigned int n, m;
 842         uint64_t i;
 843         int use_jhash = 0;
 844         int ret;
 845         uint8_t write_type;
 846         uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
 847
 848         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 849                 goto err;
 850         printf("\nTest: Hash add - key shift, Hash lookup - hit"
 851                " (non-shift-path)\n");
 852         for (m = 0; m < 2; m++) {
 853                 if (m == 1) {
 854                         printf("\n** With bulk-lookup **\n");
 855                         read_type |= BULK_LOOKUP;
 856                 }
 857                 for (n = 0; n < NUM_TEST; n++) {
 858                         unsigned int tot_lcore = rte_lcore_count();
 859                         if (tot_lcore < rwc_core_cnt[n] + 1)
 860                                 goto finish;
 861
 862                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 863
 864                         __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
 865                         __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
 866
 867                         rte_hash_reset(tbl_rwc_test_param.h);
 868                         writer_done = 0;
 869                         write_type = WRITE_NO_KEY_SHIFT;
 870                         if (write_keys(write_type) < 0)
 871                                 goto err;
 872                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 873                                 rte_eal_remote_launch(test_rwc_reader,
 874                                                 (void *)(uintptr_t)read_type,
 875                                                         enabled_core_ids[i]);
 876                         write_type = WRITE_KEY_SHIFT;
 877                         ret = write_keys(write_type);
 878                         writer_done = 1;
 879
 880                         if (ret < 0)
 881                                 goto err;
 882                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 883                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 884                                         goto err;
 885
 886                         unsigned long long cycles_per_lookup =
 887                                 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
 888                                 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
 889                         rwc_perf_results->w_ks_r_hit_nsp[m][n]
 890                                                 = cycles_per_lookup;
 891                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 892                 }
 893         }
 894
 895 finish:
 896         rte_hash_free(tbl_rwc_test_param.h);
 897         return 0;
 898
 899 err:
 900         rte_eal_mp_wait_lcore();
 901         rte_hash_free(tbl_rwc_test_param.h);
 902         return -1;
 903 }
 904
 905 /*
 906  * Test lookup perf:
 907  * Reader(s) lookup keys present in the table and likely on the shift-path while
 908  * 'Main' thread adds keys causing key-shifts.
 909  */
 910 static int
 911 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 912                                 int htm, int ext_bkt)
 913 {
 914         unsigned int n, m;
 915         uint64_t i;
 916         int use_jhash = 0;
 917         int ret;
 918         uint8_t write_type;
 919         uint8_t read_type = READ_PASS_SHIFT_PATH;
 920
 921         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 922                 goto err;
 923         printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
 924                "\n");
 925
 926         for (m = 0; m < 2; m++) {
 927                 if (m == 1) {
 928                         printf("\n** With bulk-lookup **\n");
 929                         read_type |= BULK_LOOKUP;
 930                 }
 931                 for (n = 0; n < NUM_TEST; n++) {
 932                         unsigned int tot_lcore = rte_lcore_count();
 933                         if (tot_lcore < rwc_core_cnt[n] + 1)
 934                                 goto finish;
 935
 936                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 937
 938                         __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
 939                         __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
 940
 941                         rte_hash_reset(tbl_rwc_test_param.h);
 942                         writer_done = 0;
 943                         write_type = WRITE_NO_KEY_SHIFT;
 944                         if (write_keys(write_type) < 0)
 945                                 goto err;
 946                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 947                                 rte_eal_remote_launch(test_rwc_reader,
 948                                                 (void *)(uintptr_t)read_type,
 949                                                 enabled_core_ids[i]);
 950                         write_type = WRITE_KEY_SHIFT;
 951                         ret = write_keys(write_type);
 952                         writer_done = 1;
 953
 954                         if (ret < 0)
 955                                 goto err;
 956                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 957                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 958                                         goto err;
 959
 960                         unsigned long long cycles_per_lookup =
 961                                 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
 962                                 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
 963                         rwc_perf_results->w_ks_r_hit_sp[m][n]
 964                                                 = cycles_per_lookup;
 965                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 966                 }
 967         }
 968
 969 finish:
 970         rte_hash_free(tbl_rwc_test_param.h);
 971         return 0;
 972
 973 err:
 974         rte_eal_mp_wait_lcore();
 975         rte_hash_free(tbl_rwc_test_param.h);
 976         return -1;
 977 }
 978
 979 /*
 980  * Test lookup perf:
 981  * Reader(s) lookup keys absent in the table while
 982  * 'Main' thread adds keys causing key-shifts.
 983  */
 984 static int
 985 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 986                              htm, int ext_bkt)
 987 {
 988         unsigned int n, m;
 989         uint64_t i;
 990         int use_jhash = 0;
 991         int ret;
 992         uint8_t write_type;
 993         uint8_t read_type = READ_FAIL;
 994
 995         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 996                 goto err;
 997         printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
 998         for (m = 0; m < 2; m++) {
 999                 if (m == 1) {
1000                         printf("\n** With bulk-lookup **\n");
1001                         read_type |= BULK_LOOKUP;
1002                 }
1003                 for (n = 0; n < NUM_TEST; n++) {
1004                         unsigned int tot_lcore = rte_lcore_count();
1005                         if (tot_lcore < rwc_core_cnt[n] + 1)
1006                                 goto finish;
1007
1008                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1009
1010                         __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1011                         __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
1012
1013                         rte_hash_reset(tbl_rwc_test_param.h);
1014                         writer_done = 0;
1015                         write_type = WRITE_NO_KEY_SHIFT;
1016                         if (write_keys(write_type) < 0)
1017                                 goto err;
1018                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1019                                 rte_eal_remote_launch(test_rwc_reader,
1020                                                 (void *)(uintptr_t)read_type,
1021                                                         enabled_core_ids[i]);
1022                         write_type = WRITE_KEY_SHIFT;
1023                         ret = write_keys(write_type);
1024                         writer_done = 1;
1025
1026                         if (ret < 0)
1027                                 goto err;
1028                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1029                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1030                                         goto err;
1031
1032                         unsigned long long cycles_per_lookup =
1033                                 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
1034                                 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
1035                         rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1036                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1037                 }
1038         }
1039
1040 finish:
1041         rte_hash_free(tbl_rwc_test_param.h);
1042         return 0;
1043
1044 err:
1045         rte_eal_mp_wait_lcore();
1046         rte_hash_free(tbl_rwc_test_param.h);
1047         return -1;
1048 }
1049
1050 /*
1051  * Test lookup perf for multi-writer:
1052  * Reader(s) lookup keys present in the table and likely on the shift-path while
1053  * Writers add keys causing key-shiftsi.
1054  * Writers are running in parallel, on different data plane cores.
1055  */
1056 static int
1057 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1058                            int htm, int ext_bkt)
1059 {
1060         unsigned int n, m, k;
1061         uint64_t i;
1062         int use_jhash = 0;
1063         uint8_t write_type;
1064         uint8_t read_type = READ_PASS_SHIFT_PATH;
1065
1066         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1067                 goto err;
1068         printf("\nTest: Multi-add-lookup\n");
1069         uint8_t pos_core;
1070         for (m = 1; m < NUM_TEST; m++) {
1071                 /* Calculate keys added by each writer */
1072                 tbl_rwc_test_param.single_insert =
1073                         tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1074                 for (k = 0; k < 2; k++) {
1075                         if (k == 1) {
1076                                 printf("\n** With bulk-lookup **\n");
1077                                 read_type |= BULK_LOOKUP;
1078                         }
1079                         for (n = 0; n < NUM_TEST; n++) {
1080                                 unsigned int tot_lcore  = rte_lcore_count();
1081                                 if (tot_lcore < (rwc_core_cnt[n] +
1082                                      rwc_core_cnt[m] + 1))
1083                                         goto finish;
1084
1085                                 printf("\nNumber of writers: %u",
1086                                        rwc_core_cnt[m]);
1087                                 printf("\nNumber of readers: %u\n",
1088                                        rwc_core_cnt[n]);
1089
1090                                 __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1091                                 __atomic_store_n(&gread_cycles, 0,
1092                                                  __ATOMIC_RELAXED);
1093
1094                                 rte_hash_reset(tbl_rwc_test_param.h);
1095                                 writer_done = 0;
1096                                 write_type = WRITE_NO_KEY_SHIFT;
1097                                 if (write_keys(write_type) < 0)
1098                                         goto err;
1099
1100                                 /* Launch reader(s) */
1101                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1102                                         rte_eal_remote_launch(test_rwc_reader,
1103                                                 (void *)(uintptr_t)read_type,
1104                                                 enabled_core_ids[i]);
1105                                 write_type = WRITE_KEY_SHIFT;
1106                                 pos_core = 0;
1107
1108                                 /* Launch writers */
1109                                 for (; i <= rwc_core_cnt[m]
1110                                      + rwc_core_cnt[n]; i++) {
1111                                         rte_eal_remote_launch
1112                                                 (test_rwc_multi_writer,
1113                                                 (void *)(uintptr_t)pos_core,
1114                                                 enabled_core_ids[i]);
1115                                         pos_core++;
1116                                 }
1117
1118                                 /* Wait for writers to complete */
1119                                 for (i = rwc_core_cnt[n] + 1;
1120                                      i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1121                                      i++)
1122                                         rte_eal_wait_lcore(enabled_core_ids[i]);
1123
1124                                 writer_done = 1;
1125
1126                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1127                                         if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1128                                                 goto err;
1129
1130                                 unsigned long long cycles_per_lookup =
1131                                         __atomic_load_n(&gread_cycles,
1132                                                         __ATOMIC_RELAXED) /
1133                                         __atomic_load_n(&greads,
1134                                                           __ATOMIC_RELAXED);
1135                                 rwc_perf_results->multi_rw[m][k][n]
1136                                         = cycles_per_lookup;
1137                                 printf("Cycles per lookup: %llu\n",
1138                                        cycles_per_lookup);
1139                         }
1140                 }
1141         }
1142
1143 finish:
1144         rte_hash_free(tbl_rwc_test_param.h);
1145         return 0;
1146
1147 err:
1148         rte_eal_mp_wait_lcore();
1149         rte_hash_free(tbl_rwc_test_param.h);
1150         return -1;
1151 }
1152
1153 /*
1154  * Test lookup perf:
1155  * Reader(s) lookup keys present in the extendable bkt.
1156  */
1157 static int
1158 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1159                                 int rwc_lf, int htm, int ext_bkt)
1160 {
1161         unsigned int n, m;
1162         uint64_t i;
1163         int use_jhash = 0;
1164         uint8_t write_type;
1165         uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1166
1167         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1168                 goto err;
1169         printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1170         for (m = 0; m < 2; m++) {
1171                 if (m == 1) {
1172                         printf("\n** With bulk-lookup **\n");
1173                         read_type |= BULK_LOOKUP;
1174                 }
1175                 for (n = 0; n < NUM_TEST; n++) {
1176                         unsigned int tot_lcore = rte_lcore_count();
1177                         if (tot_lcore < rwc_core_cnt[n] + 1)
1178                                 goto finish;
1179
1180                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1181
1182                         __atomic_store_n(&greads, 0, __ATOMIC_RELAXED);
1183                         __atomic_store_n(&gread_cycles, 0, __ATOMIC_RELAXED);
1184
1185                         rte_hash_reset(tbl_rwc_test_param.h);
1186                         write_type = WRITE_NO_KEY_SHIFT;
1187                         if (write_keys(write_type) < 0)
1188                                 goto err;
1189                         write_type = WRITE_KEY_SHIFT;
1190                         if (write_keys(write_type) < 0)
1191                                 goto err;
1192                         writer_done = 0;
1193                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1194                                 rte_eal_remote_launch(test_rwc_reader,
1195                                                 (void *)(uintptr_t)read_type,
1196                                                         enabled_core_ids[i]);
1197                         for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1198                              i++) {
1199                                 if (rte_hash_del_key(tbl_rwc_test_param.h,
1200                                         tbl_rwc_test_param.keys_ks_extbkt + i)
1201                                                         < 0) {
1202                                         printf("Delete Failed: %u\n",
1203                                         tbl_rwc_test_param.keys_ks_extbkt[i]);
1204                                         goto err;
1205                                 }
1206                         }
1207                         writer_done = 1;
1208
1209                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1210                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1211                                         goto err;
1212
1213                         unsigned long long cycles_per_lookup =
1214                                 __atomic_load_n(&gread_cycles, __ATOMIC_RELAXED)
1215                                 / __atomic_load_n(&greads, __ATOMIC_RELAXED);
1216                         rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1217                                                 = cycles_per_lookup;
1218                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1219                 }
1220         }
1221
1222 finish:
1223         rte_hash_free(tbl_rwc_test_param.h);
1224         return 0;
1225
1226 err:
1227         rte_eal_mp_wait_lcore();
1228         rte_hash_free(tbl_rwc_test_param.h);
1229         return -1;
1230 }
1231
1232 static struct rte_rcu_qsbr *rv;
1233
1234 /*
1235  * Reader thread using rte_hash data structure with RCU
1236  */
1237 static int
1238 test_hash_rcu_qsbr_reader(void *arg)
1239 {
1240         unsigned int i, j;
1241         uint32_t num_keys = tbl_rwc_test_param.count_keys_no_ks
1242                                 - QSBR_REPORTING_INTERVAL;
1243         uint32_t *keys = tbl_rwc_test_param.keys_no_ks;
1244         uint32_t lcore_id = rte_lcore_id();
1245         RTE_SET_USED(arg);
1246
1247         (void)rte_rcu_qsbr_thread_register(rv, lcore_id);
1248         rte_rcu_qsbr_thread_online(rv, lcore_id);
1249         do {
1250                 for (i = 0; i < num_keys; i += j) {
1251                         for (j = 0; j < QSBR_REPORTING_INTERVAL; j++)
1252                                 rte_hash_lookup(tbl_rwc_test_param.h,
1253                                                 keys + i + j);
1254                         /* Update quiescent state counter */
1255                         rte_rcu_qsbr_quiescent(rv, lcore_id);
1256                 }
1257         } while (!writer_done);
1258         rte_rcu_qsbr_thread_offline(rv, lcore_id);
1259         (void)rte_rcu_qsbr_thread_unregister(rv, lcore_id);
1260
1261         return 0;
1262 }
1263
1264 /*
1265  * Writer thread using rte_hash data structure with RCU
1266  */
1267 static int
1268 test_hash_rcu_qsbr_writer(void *arg)
1269 {
1270         uint32_t i, offset;
1271         uint64_t begin, cycles;
1272         uint8_t pos_core = (uint32_t)((uintptr_t)arg);
1273         offset = pos_core * tbl_rwc_test_param.single_insert;
1274
1275         begin = rte_rdtsc_precise();
1276         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) {
1277                 /* Delete element from the shared data structure */
1278                 rte_hash_del_key(tbl_rwc_test_param.h,
1279                                         tbl_rwc_test_param.keys_no_ks + i);
1280                 rte_hash_add_key(tbl_rwc_test_param.h,
1281                                 tbl_rwc_test_param.keys_no_ks + i);
1282         }
1283         cycles = rte_rdtsc_precise() - begin;
1284         __atomic_fetch_add(&gwrite_cycles, cycles, __ATOMIC_RELAXED);
1285         __atomic_fetch_add(&gwrites, tbl_rwc_test_param.single_insert,
1286                            __ATOMIC_RELAXED);
1287         return 0;
1288 }
1289
1290 /*
1291  * Writer perf test with RCU QSBR in DQ mode:
1292  * Writer(s) delete and add keys in the table.
1293  * Readers lookup keys in the hash table
1294  */
1295 static int
1296 test_hash_rcu_qsbr_writer_perf(struct rwc_perf *rwc_perf_results, int rwc_lf,
1297                                 int htm, int ext_bkt)
1298 {
1299         unsigned int n;
1300         uint64_t i;
1301         uint8_t write_type;
1302         int use_jhash = 0;
1303         struct rte_hash_rcu_config rcu_config = {0};
1304         uint32_t sz;
1305         uint8_t pos_core;
1306
1307         printf("\nTest: Writer perf with integrated RCU\n");
1308
1309         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1310                 goto err;
1311
1312         sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
1313         rv = (struct rte_rcu_qsbr *)rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE);
1314         rcu_config.v = rv;
1315
1316         if (rte_hash_rcu_qsbr_add(tbl_rwc_test_param.h, &rcu_config) < 0) {
1317                 printf("RCU init in hash failed\n");
1318                 goto err;
1319         }
1320
1321         for (n = 0; n < NUM_TEST; n++) {
1322                 unsigned int tot_lcore = rte_lcore_count();
1323                 if (tot_lcore < rwc_core_cnt[n] + 3)
1324                         goto finish;
1325
1326                 /* Calculate keys added by each writer */
1327                 tbl_rwc_test_param.single_insert =
1328                         tbl_rwc_test_param.count_keys_no_ks /
1329                                 rwc_core_cnt[n];
1330                 printf("\nNumber of writers: %u\n", rwc_core_cnt[n]);
1331
1332                 __atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
1333                 __atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
1334
1335                 rte_hash_reset(tbl_rwc_test_param.h);
1336                 rte_rcu_qsbr_init(rv, RTE_MAX_LCORE);
1337
1338                 write_type = WRITE_NO_KEY_SHIFT;
1339                 if (write_keys(write_type) < 0)
1340                         goto err;
1341                 write_type = WRITE_KEY_SHIFT;
1342                 if (write_keys(write_type) < 0)
1343                         goto err;
1344
1345                 /* Launch 2 readers */
1346                 for (i = 1; i <= 2; i++)
1347                         rte_eal_remote_launch(test_hash_rcu_qsbr_reader, NULL,
1348                                               enabled_core_ids[i]);
1349                 pos_core = 0;
1350                 /* Launch writer(s) */
1351                 for (; i <= rwc_core_cnt[n] + 2; i++) {
1352                         rte_eal_remote_launch(test_hash_rcu_qsbr_writer,
1353                                 (void *)(uintptr_t)pos_core,
1354                                 enabled_core_ids[i]);
1355                         pos_core++;
1356                 }
1357
1358                 /* Wait for writers to complete */
1359                 for (i = 3; i <= rwc_core_cnt[n] + 2; i++)
1360                         rte_eal_wait_lcore(enabled_core_ids[i]);
1361
1362                 writer_done = 1;
1363
1364                 /* Wait for readers to complete */
1365                 rte_eal_mp_wait_lcore();
1366
1367                 unsigned long long cycles_per_write_operation =
1368                         __atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
1369                         __atomic_load_n(&gwrites, __ATOMIC_RELAXED);
1370                 rwc_perf_results->writer_add_del[n]
1371                                         = cycles_per_write_operation;
1372                 printf("Cycles per write operation: %llu\n",
1373                                 cycles_per_write_operation);
1374         }
1375
1376 finish:
1377         rte_hash_free(tbl_rwc_test_param.h);
1378         rte_free(rv);
1379         return 0;
1380
1381 err:
1382         writer_done = 1;
1383         rte_eal_mp_wait_lcore();
1384         rte_hash_free(tbl_rwc_test_param.h);
1385         rte_free(rv);
1386         return -1;
1387 }
1388
1389 static int
1390 test_hash_readwrite_lf_perf_main(void)
1391 {
1392         /*
1393          * Variables used to choose different tests.
1394          * rwc_lf indicates if read-write concurrency lock-free support is
1395          * enabled.
1396          * htm indicates if Hardware transactional memory support is enabled.
1397          */
1398         int rwc_lf = 0;
1399         int htm;
1400         int ext_bkt = 0;
1401
1402         if (rte_lcore_count() < 2) {
1403                 printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n");
1404                 return TEST_SKIPPED;
1405         }
1406
1407         setlocale(LC_NUMERIC, "");
1408
1409         /* Reset tbl_rwc_test_param to discard values from previous run */
1410         memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param));
1411
1412         if (rte_tm_supported())
1413                 htm = 1;
1414         else
1415                 htm = 0;
1416
1417         if (generate_keys() != 0)
1418                 return -1;
1419         if (get_enabled_cores_list() != 0)
1420                 return -1;
1421
1422         if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1423                 rwc_lf = 1;
1424                 ext_bkt = 1;
1425                 printf("Test lookup with read-write concurrency lock free support"
1426                        " enabled\n");
1427                 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1428                                                         htm, ext_bkt) < 0)
1429                         return -1;
1430                 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1431                                                         htm, ext_bkt) < 0)
1432                         return -1;
1433                 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1434                                                         htm, ext_bkt) < 0)
1435                         return -1;
1436                 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1437                                                         htm, ext_bkt) < 0)
1438                         return -1;
1439                 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1440                                                  ext_bkt) < 0)
1441                         return -1;
1442                 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1443                                                ext_bkt) < 0)
1444                         return -1;
1445                 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1446                                                         htm, ext_bkt) < 0)
1447                         return -1;
1448                 if (test_hash_rcu_qsbr_writer_perf(&rwc_lf_results, rwc_lf,
1449                                                    htm, ext_bkt) < 0)
1450                         return -1;
1451         }
1452         printf("\nTest lookup with read-write concurrency lock free support"
1453                " disabled\n");
1454         rwc_lf = 0;
1455         if (!htm) {
1456                 printf("With HTM Disabled\n");
1457                 if (!RUN_WITH_HTM_DISABLED) {
1458                         printf("Enable RUN_WITH_HTM_DISABLED to test with"
1459                                " lock-free disabled");
1460                         goto results;
1461                 }
1462         } else
1463                 printf("With HTM Enabled\n");
1464         if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1465                                            ext_bkt) < 0)
1466                 return -1;
1467         if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1468                                                 ext_bkt) < 0)
1469                 return -1;
1470         if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1471                                                 htm, ext_bkt) < 0)
1472                 return -1;
1473         if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1474                                                 ext_bkt) < 0)
1475                 return -1;
1476         if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1477                                          ext_bkt) < 0)
1478                 return -1;
1479         if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1480                                                         ext_bkt) < 0)
1481                 return -1;
1482         if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1483                                                 htm, ext_bkt) < 0)
1484                 return -1;
1485 results:
1486         printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1487         int i, j, k;
1488         for (j = 0; j < 2; j++) {
1489                 if (j == 1)
1490                         printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1491                                "**********#######\n\n");
1492                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1493                         "\t\t\t\t_________________\n");
1494                 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1495                        "\t\t\tCycles per lookup\n");
1496                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1497                        "\t\t\t_________________\n");
1498                 for (i = 0; i < NUM_TEST; i++) {
1499                         printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1500                         printf("Enabled\t\t");
1501                         printf("N/A\t\t");
1502                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1503                                 "%u\n\t\t\t\t\t\t\t\t",
1504                                 rwc_lf_results.w_no_ks_r_hit[j][i]);
1505                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1506                                 "%u\n\t\t\t\t\t\t\t\t",
1507                                 rwc_lf_results.w_no_ks_r_miss[j][i]);
1508                         printf("Hash add - key-shifts, lookup - hit"
1509                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1510                                rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1511                         printf("Hash add - key-shifts, lookup - hit "
1512                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1513                                rwc_lf_results.w_ks_r_hit_sp[j][i]);
1514                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1515                                 "%u\n\t\t\t\t\t\t\t\t",
1516                                 rwc_lf_results.w_ks_r_miss[j][i]);
1517                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1518                                 "%u\n\n\t\t\t\t",
1519                                 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1520
1521                         printf("Disabled\t");
1522                         if (htm)
1523                                 printf("Enabled\t\t");
1524                         else
1525                                 printf("Disabled\t");
1526                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1527                                 "%u\n\t\t\t\t\t\t\t\t",
1528                                 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1529                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1530                                 "%u\n\t\t\t\t\t\t\t\t",
1531                                 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1532                         printf("Hash add - key-shifts, lookup - hit "
1533                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1534                                rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1535                         printf("Hash add - key-shifts, lookup - hit "
1536                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1537                                rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1538                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1539                                "%u\n\t\t\t\t\t\t\t\t",
1540                                rwc_non_lf_results.w_ks_r_miss[j][i]);
1541                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1542                                 "%u\n",
1543                                 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1544
1545                         printf("_______\t\t_______\t\t_________\t___\t\t"
1546                                "_________\t\t\t\t\t\t_________________\n");
1547                 }
1548
1549                 for (i = 1; i < NUM_TEST; i++) {
1550                         for (k = 0; k < NUM_TEST; k++) {
1551                                 printf("%u", rwc_core_cnt[i]);
1552                                 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1553                                 printf("Enabled\t\t");
1554                                 printf("N/A\t\t");
1555                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1556                                        "\t\t",
1557                                        rwc_lf_results.multi_rw[i][j][k]);
1558                                 printf("Disabled\t");
1559                                 if (htm)
1560                                         printf("Enabled\t\t");
1561                                 else
1562                                         printf("Disabled\t");
1563                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1564                                        rwc_non_lf_results.multi_rw[i][j][k]);
1565
1566                                 printf("_______\t\t_______\t\t_________\t___"
1567                                        "\t\t_________\t\t\t\t\t\t"
1568                                        "_________________\n");
1569                         }
1570                 }
1571         }
1572         rte_free(tbl_rwc_test_param.keys);
1573         rte_free(tbl_rwc_test_param.keys_no_ks);
1574         rte_free(tbl_rwc_test_param.keys_ks);
1575         rte_free(tbl_rwc_test_param.keys_absent);
1576         rte_free(tbl_rwc_test_param.keys_shift_path);
1577         rte_free(tbl_rwc_test_param.keys_non_shift_path);
1578         rte_free(tbl_rwc_test_param.keys_ext_bkt);
1579         rte_free(tbl_rwc_test_param.keys_ks_extbkt);
1580         return 0;
1581 }
1582
1583 REGISTER_TEST_COMMAND(hash_readwrite_lf_perf_autotest,
1584         test_hash_readwrite_lf_perf_main);