app/test/test_hash_readwrite_lf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Arm Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
  20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
  21 #endif
  22
  23 #define BULK_LOOKUP_SIZE 32
  24
  25 #define RUN_WITH_HTM_DISABLED 0
  26
  27 #if (RUN_WITH_HTM_DISABLED)
  28
  29 #define TOTAL_ENTRY (5*1024)
  30 #define TOTAL_INSERT (5*1024)
  31
  32 #else
  33
  34 #define TOTAL_ENTRY (4*1024*1024)
  35 #define TOTAL_INSERT (4*1024*1024)
  36
  37 #endif
  38
  39 #define READ_FAIL 1
  40 #define READ_PASS_NO_KEY_SHIFTS 2
  41 #define READ_PASS_SHIFT_PATH 4
  42 #define READ_PASS_NON_SHIFT_PATH 8
  43 #define BULK_LOOKUP 16
  44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
  45
  46 #define WRITE_NO_KEY_SHIFT 0
  47 #define WRITE_KEY_SHIFT 1
  48 #define WRITE_EXT_BKT 2
  49
  50 #define NUM_TEST 3
  51 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
  52
  53 struct rwc_perf {
  54         uint32_t w_no_ks_r_hit[2][NUM_TEST];
  55         uint32_t w_no_ks_r_miss[2][NUM_TEST];
  56         uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
  57         uint32_t w_ks_r_hit_sp[2][NUM_TEST];
  58         uint32_t w_ks_r_miss[2][NUM_TEST];
  59         uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
  60         uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
  61 };
  62
  63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
  64
  65 struct {
  66         uint32_t *keys;
  67         uint32_t *keys_no_ks;
  68         uint32_t *keys_ks;
  69         uint32_t *keys_absent;
  70         uint32_t *keys_shift_path;
  71         uint32_t *keys_non_shift_path;
  72         uint32_t *keys_ext_bkt;
  73         uint32_t *keys_ks_extbkt;
  74         uint32_t count_keys_no_ks;
  75         uint32_t count_keys_ks;
  76         uint32_t count_keys_absent;
  77         uint32_t count_keys_shift_path;
  78         uint32_t count_keys_non_shift_path;
  79         uint32_t count_keys_extbkt;
  80         uint32_t count_keys_ks_extbkt;
  81         uint32_t single_insert;
  82         struct rte_hash *h;
  83 } tbl_rwc_test_param;
  84
  85 static rte_atomic64_t gread_cycles;
  86 static rte_atomic64_t greads;
  87
  88 static volatile uint8_t writer_done;
  89 static volatile uint8_t multi_writer_done[4];
  90
  91 uint16_t enabled_core_ids[RTE_MAX_LCORE];
  92
  93 uint8_t *scanned_bkts;
  94
  95 static inline uint16_t
  96 get_short_sig(const hash_sig_t hash)
  97 {
  98         return hash >> 16;
  99 }
 100
 101 static inline uint32_t
 102 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 103                       const hash_sig_t hash)
 104 {
 105         uint32_t num_buckets;
 106         uint32_t bucket_bitmask;
 107         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 108         bucket_bitmask = num_buckets - 1;
 109         return hash & bucket_bitmask;
 110 }
 111
 112 static inline uint32_t
 113 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 114                         uint32_t cur_bkt_idx, uint16_t sig)
 115 {
 116         uint32_t num_buckets;
 117         uint32_t bucket_bitmask;
 118         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 119         bucket_bitmask = num_buckets - 1;
 120         return (cur_bkt_idx ^ sig) & bucket_bitmask;
 121 }
 122
 123
 124 static inline int
 125 get_enabled_cores_list(void)
 126 {
 127         uint32_t i = 0;
 128         uint16_t core_id;
 129         uint32_t max_cores = rte_lcore_count();
 130         for (core_id = 0; core_id < RTE_MAX_LCORE && i < max_cores; core_id++) {
 131                 if (rte_lcore_is_enabled(core_id)) {
 132                         enabled_core_ids[i] = core_id;
 133                         i++;
 134                 }
 135         }
 136
 137         if (i != max_cores) {
 138                 printf("Number of enabled cores in list is different from "
 139                         "number given by rte_lcore_count()\n");
 140                 return -1;
 141         }
 142         return 0;
 143 }
 144
 145 static inline int
 146 check_bucket(uint32_t bkt_idx, uint32_t key)
 147 {
 148         uint32_t iter;
 149         uint32_t prev_iter;
 150         uint32_t diff;
 151         uint32_t count = 0;
 152         const void *next_key;
 153         void *next_data;
 154
 155         /* Temporary bucket to hold the keys */
 156         uint32_t keys_in_bkt[8];
 157
 158         iter = bkt_idx * 8;
 159         prev_iter = iter;
 160         while (rte_hash_iterate(tbl_rwc_test_param.h,
 161                         &next_key, &next_data, &iter) >= 0) {
 162
 163                 /* Check for duplicate entries */
 164                 if (*(const uint32_t *)next_key == key)
 165                         return 1;
 166
 167                 /* Identify if there is any free entry in the bucket */
 168                 diff = iter - prev_iter;
 169                 if (diff > 1)
 170                         break;
 171
 172                 prev_iter = iter;
 173                 keys_in_bkt[count] = *(const uint32_t *)next_key;
 174                 count++;
 175
 176                 /* All entries in the bucket are occupied */
 177                 if (count == 8) {
 178
 179                         /*
 180                          * Check if bucket was not scanned before, to avoid
 181                          * duplicate keys.
 182                          */
 183                         if (scanned_bkts[bkt_idx] == 0) {
 184                                 /*
 185                                  * Since this bucket (pointed to by bkt_idx) is
 186                                  * full, it is likely that key(s) in this
 187                                  * bucket will be on the shift path, when
 188                                  * collision occurs. Thus, add it to
 189                                  * keys_shift_path.
 190                                  */
 191                                 memcpy(tbl_rwc_test_param.keys_shift_path +
 192                                         tbl_rwc_test_param.count_keys_shift_path
 193                                         , keys_in_bkt, 32);
 194                                 tbl_rwc_test_param.count_keys_shift_path += 8;
 195                                 scanned_bkts[bkt_idx] = 1;
 196                         }
 197                         return -1;
 198                 }
 199         }
 200         return 0;
 201 }
 202
 203 static int
 204 generate_keys(void)
 205 {
 206         uint32_t *keys = NULL;
 207         uint32_t *keys_no_ks = NULL;
 208         uint32_t *keys_ks = NULL;
 209         uint32_t *keys_absent = NULL;
 210         uint32_t *keys_non_shift_path = NULL;
 211         uint32_t *keys_ext_bkt = NULL;
 212         uint32_t *keys_ks_extbkt = NULL;
 213         uint32_t *found = NULL;
 214         uint32_t count_keys_no_ks = 0;
 215         uint32_t count_keys_ks = 0;
 216         uint32_t count_keys_extbkt = 0;
 217         uint32_t i;
 218
 219         /*
 220          * keys will consist of a) keys whose addition to the hash table
 221          * will result in shifting of the existing keys to their alternate
 222          * locations b) keys whose addition to the hash table will not result
 223          * in shifting of the existing keys.
 224          */
 225         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 226         if (keys == NULL) {
 227                 printf("RTE_MALLOC failed\n");
 228                 goto err;
 229         }
 230
 231         /*
 232          * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
 233          * will NOT result in shifting of the existing keys to their alternate
 234          * locations. Roughly around 900K keys.
 235          */
 236         keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 237         if (keys_no_ks == NULL) {
 238                 printf("RTE_MALLOC failed\n");
 239                 goto err;
 240         }
 241
 242         /*
 243          * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
 244          * result in shifting of the existing keys to their alternate locations.
 245          * Roughly around 146K keys. There might be repeating keys. More code is
 246          * required to filter out these keys which will complicate the test case
 247          */
 248         keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 249         if (keys_ks == NULL) {
 250                 printf("RTE_MALLOC failed\n");
 251                 goto err;
 252         }
 253
 254         /* Used to identify keys not inserted in the hash table */
 255         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 256         if (found == NULL) {
 257                 printf("RTE_MALLOC failed\n");
 258                 goto err;
 259         }
 260
 261         /*
 262          * This consist of keys not inserted to the hash table.
 263          * Used to test perf of lookup on keys that do not exist in the table.
 264          */
 265         keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 266         if (keys_absent == NULL) {
 267                 printf("RTE_MALLOC failed\n");
 268                 goto err;
 269         }
 270
 271         /*
 272          * This consist of keys which are likely to be on the shift
 273          * path (i.e. being moved to alternate location), when collision occurs
 274          * on addition of a key to an already full primary bucket.
 275          * Used to test perf of lookup on keys that are on the shift path.
 276          */
 277         tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
 278                                                         TOTAL_INSERT, 0);
 279         if (tbl_rwc_test_param.keys_shift_path == NULL) {
 280                 printf("RTE_MALLOC failed\n");
 281                 goto err;
 282         }
 283
 284         /*
 285          * This consist of keys which are never on the shift
 286          * path (i.e. being moved to alternate location), when collision occurs
 287          * on addition of a key to an already full primary bucket.
 288          * Used to test perf of lookup on keys that are not on the shift path.
 289          */
 290         keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
 291                                          0);
 292         if (keys_non_shift_path == NULL) {
 293                 printf("RTE_MALLOC failed\n");
 294                 goto err;
 295         }
 296
 297         /*
 298          * This consist of keys which will be stored in extended buckets
 299          */
 300         keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 301         if (keys_ext_bkt == NULL) {
 302                 printf("RTE_MALLOC failed\n");
 303                 goto err;
 304         }
 305
 306         /*
 307          * This consist of keys which when deleted causes shifting of keys
 308          * in extended buckets to respective secondary buckets
 309          */
 310         keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 311         if (keys_ks_extbkt == NULL) {
 312                 printf("RTE_MALLOC failed\n");
 313                 goto err;
 314         }
 315
 316         hash_sig_t sig;
 317         uint32_t prim_bucket_idx;
 318         uint32_t sec_bucket_idx;
 319         uint16_t short_sig;
 320         uint32_t num_buckets;
 321         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 322         int ret;
 323
 324         /*
 325          * Used to mark bkts in which at least one key was shifted to its
 326          * alternate location
 327          */
 328         scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
 329         if (scanned_bkts == NULL) {
 330                 printf("RTE_MALLOC failed\n");
 331                 goto err;
 332         }
 333
 334         tbl_rwc_test_param.keys = keys;
 335         tbl_rwc_test_param.keys_no_ks = keys_no_ks;
 336         tbl_rwc_test_param.keys_ks = keys_ks;
 337         tbl_rwc_test_param.keys_absent = keys_absent;
 338         tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
 339         tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
 340         tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
 341         /* Generate keys by adding previous two keys, neglect overflow */
 342         printf("Generating keys...\n");
 343         keys[0] = 0;
 344         keys[1] = 1;
 345         for (i = 2; i < TOTAL_INSERT; i++)
 346                 keys[i] = keys[i-1] + keys[i-2];
 347
 348         /* Segregate keys into keys_no_ks and keys_ks */
 349         for (i = 0; i < TOTAL_INSERT; i++) {
 350                 /* Check if primary bucket has space.*/
 351                 sig = rte_hash_hash(tbl_rwc_test_param.h,
 352                                         tbl_rwc_test_param.keys+i);
 353                 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
 354                                                         sig);
 355                 ret = check_bucket(prim_bucket_idx, keys[i]);
 356                 if (ret < 0) {
 357                         /*
 358                          * Primary bucket is full, this key will result in
 359                          * shifting of the keys to their alternate locations.
 360                          */
 361                         keys_ks[count_keys_ks] = keys[i];
 362                         count_keys_ks++;
 363                 } else if (ret == 0) {
 364                         /*
 365                          * Primary bucket has space, this key will not result in
 366                          * shifting of the keys. Hence, add key to the table.
 367                          */
 368                         ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
 369                                                         keys+i,
 370                                                         (void *)((uintptr_t)i));
 371                         if (ret < 0) {
 372                                 printf("writer failed %"PRIu32"\n", i);
 373                                 break;
 374                         }
 375                         keys_no_ks[count_keys_no_ks] = keys[i];
 376                         count_keys_no_ks++;
 377                 }
 378         }
 379
 380         for (i = 0; i < count_keys_no_ks; i++) {
 381                 /*
 382                  * Identify keys in keys_no_ks with value less than
 383                  * 4M (HTM enabled) OR 5K (HTM disabled)
 384                  */
 385                 if (keys_no_ks[i] < TOTAL_INSERT)
 386                         found[keys_no_ks[i]]++;
 387         }
 388
 389         for (i = 0; i < count_keys_ks; i++) {
 390                 /*
 391                  * Identify keys in keys_ks with value less than
 392                  * 4M (HTM enabled) OR 5K (HTM disabled)
 393                  */
 394                 if (keys_ks[i] < TOTAL_INSERT)
 395                         found[keys_ks[i]]++;
 396         }
 397
 398         uint32_t count_keys_absent = 0;
 399         for (i = 0; i < TOTAL_INSERT; i++) {
 400                 /*
 401                  * Identify missing keys between 0 and
 402                  * 4M (HTM enabled) OR 5K (HTM disabled)
 403                  */
 404                 if (found[i] == 0)
 405                         keys_absent[count_keys_absent++] = i;
 406         }
 407
 408         /* Find keys that will not be on the shift path */
 409         uint32_t iter;
 410         const void *next_key;
 411         void *next_data;
 412         uint32_t count = 0;
 413         for (i = 0; i < num_buckets; i++) {
 414                 /* Check bucket for no keys shifted to alternate locations */
 415                 if (scanned_bkts[i] == 0) {
 416                         iter = i * 8;
 417                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 418                                 &next_key, &next_data, &iter) >= 0) {
 419
 420                                 /* Check if key belongs to the current bucket */
 421                                 if (i >= (iter-1)/8)
 422                                         keys_non_shift_path[count++]
 423                                                 = *(const uint32_t *)next_key;
 424                                 else
 425                                         break;
 426                         }
 427                 }
 428         }
 429
 430         tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
 431         tbl_rwc_test_param.count_keys_ks = count_keys_ks;
 432         tbl_rwc_test_param.count_keys_absent = count_keys_absent;
 433         tbl_rwc_test_param.count_keys_non_shift_path = count;
 434
 435         memset(scanned_bkts, 0, num_buckets);
 436         count = 0;
 437         /* Find keys that will be in extended buckets */
 438         for (i = 0; i < count_keys_ks; i++) {
 439                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
 440                 if (ret < 0) {
 441                         /* Key will be added to ext bkt */
 442                         keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
 443                         /* Sec bkt to be added to keys_ks_extbkt */
 444                         sig = rte_hash_hash(tbl_rwc_test_param.h,
 445                                         tbl_rwc_test_param.keys_ks + i);
 446                         prim_bucket_idx = get_prim_bucket_index(
 447                                                 tbl_rwc_test_param.h, sig);
 448                         short_sig = get_short_sig(sig);
 449                         sec_bucket_idx = get_alt_bucket_index(
 450                                                 tbl_rwc_test_param.h,
 451                                                 prim_bucket_idx, short_sig);
 452                         if (scanned_bkts[sec_bucket_idx] == 0)
 453                                 scanned_bkts[sec_bucket_idx] = 1;
 454                 }
 455         }
 456
 457         /* Find keys that will shift keys in ext bucket*/
 458         for (i = 0; i < num_buckets; i++) {
 459                 if (scanned_bkts[i] == 1) {
 460                         iter = i * 8;
 461                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 462                                 &next_key, &next_data, &iter) >= 0) {
 463                                 /* Check if key belongs to the current bucket */
 464                                 if (i >= (iter-1)/8)
 465                                         keys_ks_extbkt[count++]
 466                                                 = *(const uint32_t *)next_key;
 467                                 else
 468                                         break;
 469                         }
 470                 }
 471         }
 472
 473         tbl_rwc_test_param.count_keys_ks_extbkt = count;
 474         tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
 475
 476         printf("\nCount of keys NOT causing shifting of existing keys to "
 477         "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
 478         printf("\nCount of keys causing shifting of existing keys to alternate "
 479                 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
 480         printf("Count of absent keys that will never be added to the hash "
 481                 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
 482         printf("Count of keys likely to be on the shift path: %d\n\n",
 483                tbl_rwc_test_param.count_keys_shift_path);
 484         printf("Count of keys not likely to be on the shift path: %d\n\n",
 485                tbl_rwc_test_param.count_keys_non_shift_path);
 486         printf("Count of keys in extended buckets: %d\n\n",
 487                tbl_rwc_test_param.count_keys_extbkt);
 488         printf("Count of keys shifting keys in ext buckets: %d\n\n",
 489                tbl_rwc_test_param.count_keys_ks_extbkt);
 490
 491         rte_free(found);
 492         rte_hash_free(tbl_rwc_test_param.h);
 493         return 0;
 494
 495 err:
 496         rte_free(keys);
 497         rte_free(keys_no_ks);
 498         rte_free(keys_ks);
 499         rte_free(keys_absent);
 500         rte_free(found);
 501         rte_free(tbl_rwc_test_param.keys_shift_path);
 502         rte_free(keys_ext_bkt);
 503         rte_free(keys_ks_extbkt);
 504         rte_free(scanned_bkts);
 505         return -1;
 506 }
 507
 508 static int
 509 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
 510 {
 511         struct rte_hash *handle;
 512
 513         struct rte_hash_parameters hash_params = {
 514                 .entries = TOTAL_ENTRY,
 515                 .key_len = sizeof(uint32_t),
 516                 .hash_func_init_val = 0,
 517                 .socket_id = rte_socket_id(),
 518         };
 519
 520         if (use_jhash)
 521                 hash_params.hash_func = rte_jhash;
 522         else
 523                 hash_params.hash_func = rte_hash_crc;
 524
 525         if (rwc_lf)
 526                 hash_params.extra_flag =
 527                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
 528                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 529         else if (htm)
 530                 hash_params.extra_flag =
 531                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 532                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 533                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 534         else
 535                 hash_params.extra_flag =
 536                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 537                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 538
 539         if (ext_bkt)
 540                 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
 541
 542         hash_params.name = "tests";
 543
 544         handle = rte_hash_create(&hash_params);
 545         if (handle == NULL) {
 546                 printf("hash creation failed");
 547                 return -1;
 548         }
 549
 550         tbl_rwc_test_param.h = handle;
 551         return 0;
 552 }
 553
 554 static int
 555 test_rwc_reader(__attribute__((unused)) void *arg)
 556 {
 557         uint32_t i, j;
 558         int ret;
 559         uint64_t begin, cycles;
 560         uint32_t loop_cnt = 0;
 561         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 562         uint32_t read_cnt;
 563         uint32_t *keys;
 564         uint32_t extra_keys;
 565         int32_t *pos;
 566         void *temp_a[BULK_LOOKUP_SIZE];
 567
 568         /* Used to identify keys not inserted in the hash table */
 569         pos = rte_malloc(NULL, sizeof(uint32_t) * BULK_LOOKUP_SIZE, 0);
 570         if (pos == NULL) {
 571                 printf("RTE_MALLOC failed\n");
 572                 return -1;
 573         }
 574
 575         if (read_type & READ_FAIL) {
 576                 keys = tbl_rwc_test_param.keys_absent;
 577                 read_cnt = tbl_rwc_test_param.count_keys_absent;
 578         } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
 579                 keys = tbl_rwc_test_param.keys_no_ks;
 580                 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
 581         } else if (read_type & READ_PASS_SHIFT_PATH) {
 582                 keys = tbl_rwc_test_param.keys_shift_path;
 583                 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
 584         } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
 585                 keys = tbl_rwc_test_param.keys_ext_bkt;
 586                 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
 587         } else {
 588                 keys = tbl_rwc_test_param.keys_non_shift_path;
 589                 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
 590         }
 591
 592         extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
 593
 594         begin = rte_rdtsc_precise();
 595         do {
 596                 if (read_type & BULK_LOOKUP) {
 597                         for (i = 0; i < (read_cnt - extra_keys);
 598                              i += BULK_LOOKUP_SIZE) {
 599                                 /* Array of  pointer to the list of keys */
 600                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 601                                         temp_a[j] = keys + i + j;
 602                                 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 603                                                    (const void **)
 604                                                    ((uintptr_t)temp_a),
 605                                                    BULK_LOOKUP_SIZE, pos);
 606                                 /* Validate lookup result */
 607                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 608                                         if ((read_type & READ_FAIL &&
 609                                              pos[j] != -ENOENT) ||
 610                                             (!(read_type & READ_FAIL) &&
 611                                              pos[j] == -ENOENT)) {
 612                                                 printf("lookup failed!"
 613                                                        "%"PRIu32"\n",
 614                                                        keys[i + j]);
 615                                                 return -1;
 616                                         }
 617                         }
 618                         for (j = 0; j < extra_keys; j++)
 619                                 temp_a[j] = keys + i + j;
 620
 621                         rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 622                                            (const void **)
 623                                            ((uintptr_t)temp_a),
 624                                            extra_keys, pos);
 625                         for (j = 0; j < extra_keys; j++)
 626                                 if ((read_type & READ_FAIL &&
 627                                      pos[j] != -ENOENT) ||
 628                                     (!(read_type & READ_FAIL) &&
 629                                      pos[j] == -ENOENT)) {
 630                                         printf("lookup failed! %"PRIu32"\n",
 631                                                keys[i + j]);
 632                                         return -1;
 633                                 }
 634                 } else {
 635                         for (i = 0; i < read_cnt; i++) {
 636                                 ret = rte_hash_lookup
 637                                         (tbl_rwc_test_param.h, keys + i);
 638                                 if (((read_type & READ_FAIL) &&
 639                                      (ret != -ENOENT)) ||
 640                                     (!(read_type & READ_FAIL) &&
 641                                         ret == -ENOENT)) {
 642                                         printf("lookup failed! %"PRIu32"\n",
 643                                                keys[i]);
 644                                         return -1;
 645                                 }
 646                         }
 647                 }
 648                 loop_cnt++;
 649         } while (!writer_done);
 650
 651         cycles = rte_rdtsc_precise() - begin;
 652         rte_atomic64_add(&gread_cycles, cycles);
 653         rte_atomic64_add(&greads, read_cnt*loop_cnt);
 654         return 0;
 655 }
 656
 657 static int
 658 write_keys(uint8_t write_type)
 659 {
 660         uint32_t i;
 661         int ret;
 662         uint32_t key_cnt = 0;
 663         uint32_t *keys;
 664         if (write_type == WRITE_KEY_SHIFT) {
 665                 key_cnt = tbl_rwc_test_param.count_keys_ks;
 666                 keys = tbl_rwc_test_param.keys_ks;
 667         } else if (write_type == WRITE_NO_KEY_SHIFT) {
 668                 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
 669                 keys = tbl_rwc_test_param.keys_no_ks;
 670         } else if (write_type == WRITE_EXT_BKT) {
 671                 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
 672                 keys = tbl_rwc_test_param.keys_ext_bkt;
 673         }
 674         for (i = 0; i < key_cnt; i++) {
 675                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
 676                 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
 677                         printf("writer failed %"PRIu32"\n", i);
 678                         return -1;
 679                 }
 680         }
 681         return 0;
 682 }
 683
 684 static int
 685 test_rwc_multi_writer(__attribute__((unused)) void *arg)
 686 {
 687         uint32_t i, offset;
 688         uint32_t pos_core = (uint32_t)((uintptr_t)arg);
 689         offset = pos_core * tbl_rwc_test_param.single_insert;
 690         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
 691                 rte_hash_add_key(tbl_rwc_test_param.h,
 692                                  tbl_rwc_test_param.keys_ks + i);
 693         multi_writer_done[pos_core] = 1;
 694         return 0;
 695 }
 696
 697 /*
 698  * Test lookup perf:
 699  * Reader(s) lookup keys present in the table.
 700  */
 701 static int
 702 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 703                                 int htm, int ext_bkt)
 704 {
 705         unsigned int n, m;
 706         uint64_t i;
 707         int use_jhash = 0;
 708         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 709         uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
 710
 711         rte_atomic64_init(&greads);
 712         rte_atomic64_init(&gread_cycles);
 713
 714         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 715                 goto err;
 716         printf("\nTest: Hash add - no key-shifts, read - hit\n");
 717         for (m = 0; m < 2; m++) {
 718                 if (m == 1) {
 719                         printf("\n** With bulk-lookup **\n");
 720                         read_type |= BULK_LOOKUP;
 721                 }
 722                 for (n = 0; n < NUM_TEST; n++) {
 723                         unsigned int tot_lcore = rte_lcore_count();
 724                         if (tot_lcore < rwc_core_cnt[n] + 1)
 725                                 goto finish;
 726
 727                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 728
 729                         rte_atomic64_clear(&greads);
 730                         rte_atomic64_clear(&gread_cycles);
 731
 732                         rte_hash_reset(tbl_rwc_test_param.h);
 733                         writer_done = 0;
 734                         if (write_keys(write_type) < 0)
 735                                 goto err;
 736                         writer_done = 1;
 737                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 738                                 rte_eal_remote_launch(test_rwc_reader,
 739                                                 (void *)(uintptr_t)read_type,
 740                                                         enabled_core_ids[i]);
 741                         rte_eal_mp_wait_lcore();
 742
 743                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 744                                 if (lcore_config[i].ret < 0)
 745                                         goto err;
 746
 747                         unsigned long long cycles_per_lookup =
 748                                 rte_atomic64_read(&gread_cycles) /
 749                                 rte_atomic64_read(&greads);
 750                         rwc_perf_results->w_no_ks_r_hit[m][n]
 751                                                 = cycles_per_lookup;
 752                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 753                 }
 754         }
 755
 756 finish:
 757         rte_hash_free(tbl_rwc_test_param.h);
 758         return 0;
 759
 760 err:
 761         rte_hash_free(tbl_rwc_test_param.h);
 762         return -1;
 763 }
 764
 765 /*
 766  * Test lookup perf:
 767  * Reader(s) lookup keys absent in the table while
 768  * 'Main' thread adds with no key-shifts.
 769  */
 770 static int
 771 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 772                                 int htm, int ext_bkt)
 773 {
 774         unsigned int n, m;
 775         uint64_t i;
 776         int use_jhash = 0;
 777         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 778         uint8_t read_type = READ_FAIL;
 779         int ret;
 780
 781         rte_atomic64_init(&greads);
 782         rte_atomic64_init(&gread_cycles);
 783
 784         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 785                 goto err;
 786         printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
 787         for (m = 0; m < 2; m++) {
 788                 if (m == 1) {
 789                         printf("\n** With bulk-lookup **\n");
 790                         read_type |= BULK_LOOKUP;
 791                 }
 792                 for (n = 0; n < NUM_TEST; n++) {
 793                         unsigned int tot_lcore = rte_lcore_count();
 794                         if (tot_lcore < rwc_core_cnt[n] + 1)
 795                                 goto finish;
 796
 797                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 798
 799                         rte_atomic64_clear(&greads);
 800                         rte_atomic64_clear(&gread_cycles);
 801
 802                         rte_hash_reset(tbl_rwc_test_param.h);
 803                         writer_done = 0;
 804
 805                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 806                                 rte_eal_remote_launch(test_rwc_reader,
 807                                                 (void *)(uintptr_t)read_type,
 808                                                         enabled_core_ids[i]);
 809                         ret = write_keys(write_type);
 810                         writer_done = 1;
 811                         rte_eal_mp_wait_lcore();
 812
 813                         if (ret < 0)
 814                                 goto err;
 815                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 816                                 if (lcore_config[i].ret < 0)
 817                                         goto err;
 818
 819                         unsigned long long cycles_per_lookup =
 820                                 rte_atomic64_read(&gread_cycles) /
 821                                 rte_atomic64_read(&greads);
 822                         rwc_perf_results->w_no_ks_r_miss[m][n]
 823                                                 = cycles_per_lookup;
 824                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 825                 }
 826         }
 827
 828 finish:
 829         rte_hash_free(tbl_rwc_test_param.h);
 830         return 0;
 831
 832 err:
 833         rte_hash_free(tbl_rwc_test_param.h);
 834         return -1;
 835 }
 836
 837 /*
 838  * Test lookup perf:
 839  * Reader(s) lookup keys present in the table and not likely to be on the
 840  * shift path  while 'Main' thread adds keys causing key-shifts.
 841  */
 842 static int
 843 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 844                                     int rwc_lf, int htm, int ext_bkt)
 845 {
 846         unsigned int n, m;
 847         uint64_t i;
 848         int use_jhash = 0;
 849         int ret;
 850         uint8_t write_type;
 851         uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
 852
 853         rte_atomic64_init(&greads);
 854         rte_atomic64_init(&gread_cycles);
 855
 856         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 857                 goto err;
 858         printf("\nTest: Hash add - key shift, Hash lookup - hit"
 859                " (non-shift-path)\n");
 860         for (m = 0; m < 2; m++) {
 861                 if (m == 1) {
 862                         printf("\n** With bulk-lookup **\n");
 863                         read_type |= BULK_LOOKUP;
 864                 }
 865                 for (n = 0; n < NUM_TEST; n++) {
 866                         unsigned int tot_lcore = rte_lcore_count();
 867                         if (tot_lcore < rwc_core_cnt[n] + 1)
 868                                 goto finish;
 869
 870                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 871
 872                         rte_atomic64_clear(&greads);
 873                         rte_atomic64_clear(&gread_cycles);
 874
 875                         rte_hash_reset(tbl_rwc_test_param.h);
 876                         writer_done = 0;
 877                         write_type = WRITE_NO_KEY_SHIFT;
 878                         if (write_keys(write_type) < 0)
 879                                 goto err;
 880                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 881                                 rte_eal_remote_launch(test_rwc_reader,
 882                                                 (void *)(uintptr_t)read_type,
 883                                                         enabled_core_ids[i]);
 884                         write_type = WRITE_KEY_SHIFT;
 885                         ret = write_keys(write_type);
 886                         writer_done = 1;
 887                         rte_eal_mp_wait_lcore();
 888
 889                         if (ret < 0)
 890                                 goto err;
 891                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 892                                 if (lcore_config[i].ret < 0)
 893                                         goto err;
 894
 895                         unsigned long long cycles_per_lookup =
 896                                 rte_atomic64_read(&gread_cycles) /
 897                                 rte_atomic64_read(&greads);
 898                         rwc_perf_results->w_ks_r_hit_nsp[m][n]
 899                                                 = cycles_per_lookup;
 900                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 901                 }
 902         }
 903
 904 finish:
 905         rte_hash_free(tbl_rwc_test_param.h);
 906         return 0;
 907
 908 err:
 909         rte_hash_free(tbl_rwc_test_param.h);
 910         return -1;
 911 }
 912
 913 /*
 914  * Test lookup perf:
 915  * Reader(s) lookup keys present in the table and likely on the shift-path while
 916  * 'Main' thread adds keys causing key-shifts.
 917  */
 918 static int
 919 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 920                                 int htm, int ext_bkt)
 921 {
 922         unsigned int n, m;
 923         uint64_t i;
 924         int use_jhash = 0;
 925         int ret;
 926         uint8_t write_type;
 927         uint8_t read_type = READ_PASS_SHIFT_PATH;
 928
 929         rte_atomic64_init(&greads);
 930         rte_atomic64_init(&gread_cycles);
 931
 932         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 933                 goto err;
 934         printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
 935                "\n");
 936
 937         for (m = 0; m < 2; m++) {
 938                 if (m == 1) {
 939                         printf("\n** With bulk-lookup **\n");
 940                         read_type |= BULK_LOOKUP;
 941                 }
 942                 for (n = 0; n < NUM_TEST; n++) {
 943                         unsigned int tot_lcore = rte_lcore_count();
 944                         if (tot_lcore < rwc_core_cnt[n])
 945                                 goto finish;
 946
 947                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 948                         rte_atomic64_clear(&greads);
 949                         rte_atomic64_clear(&gread_cycles);
 950
 951                         rte_hash_reset(tbl_rwc_test_param.h);
 952                         writer_done = 0;
 953                         write_type = WRITE_NO_KEY_SHIFT;
 954                         if (write_keys(write_type) < 0)
 955                                 goto err;
 956                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 957                                 rte_eal_remote_launch(test_rwc_reader,
 958                                                 (void *)(uintptr_t)read_type,
 959                                                 enabled_core_ids[i]);
 960                         write_type = WRITE_KEY_SHIFT;
 961                         ret = write_keys(write_type);
 962                         writer_done = 1;
 963                         rte_eal_mp_wait_lcore();
 964
 965                         if (ret < 0)
 966                                 goto err;
 967                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 968                                 if (lcore_config[i].ret < 0)
 969                                         goto err;
 970
 971                         unsigned long long cycles_per_lookup =
 972                                 rte_atomic64_read(&gread_cycles) /
 973                                 rte_atomic64_read(&greads);
 974                         rwc_perf_results->w_ks_r_hit_sp[m][n]
 975                                                 = cycles_per_lookup;
 976                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 977                 }
 978         }
 979
 980 finish:
 981         rte_hash_free(tbl_rwc_test_param.h);
 982         return 0;
 983
 984 err:
 985         rte_hash_free(tbl_rwc_test_param.h);
 986         return -1;
 987 }
 988
 989 /*
 990  * Test lookup perf:
 991  * Reader(s) lookup keys absent in the table while
 992  * 'Main' thread adds keys causing key-shifts.
 993  */
 994 static int
 995 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 996                              htm, int ext_bkt)
 997 {
 998         unsigned int n, m;
 999         uint64_t i;
1000         int use_jhash = 0;
1001         int ret;
1002         uint8_t write_type;
1003         uint8_t read_type = READ_FAIL;
1004
1005         rte_atomic64_init(&greads);
1006         rte_atomic64_init(&gread_cycles);
1007
1008         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1009                 goto err;
1010         printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1011         for (m = 0; m < 2; m++) {
1012                 if (m == 1) {
1013                         printf("\n** With bulk-lookup **\n");
1014                         read_type |= BULK_LOOKUP;
1015                 }
1016                 for (n = 0; n < NUM_TEST; n++) {
1017                         unsigned int tot_lcore = rte_lcore_count();
1018                         if (tot_lcore < rwc_core_cnt[n] + 1)
1019                                 goto finish;
1020
1021                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1022
1023                         rte_atomic64_clear(&greads);
1024                         rte_atomic64_clear(&gread_cycles);
1025
1026                         rte_hash_reset(tbl_rwc_test_param.h);
1027                         writer_done = 0;
1028                         write_type = WRITE_NO_KEY_SHIFT;
1029                         if (write_keys(write_type) < 0)
1030                                 goto err;
1031                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1032                                 rte_eal_remote_launch(test_rwc_reader,
1033                                                 (void *)(uintptr_t)read_type,
1034                                                         enabled_core_ids[i]);
1035                         write_type = WRITE_KEY_SHIFT;
1036                         ret = write_keys(write_type);
1037                         writer_done = 1;
1038                         rte_eal_mp_wait_lcore();
1039
1040                         if (ret < 0)
1041                                 goto err;
1042                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1043                                 if (lcore_config[i].ret < 0)
1044                                         goto err;
1045
1046                         unsigned long long cycles_per_lookup =
1047                                 rte_atomic64_read(&gread_cycles) /
1048                                 rte_atomic64_read(&greads);
1049                         rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1050                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1051                 }
1052         }
1053
1054 finish:
1055         rte_hash_free(tbl_rwc_test_param.h);
1056         return 0;
1057
1058 err:
1059         rte_hash_free(tbl_rwc_test_param.h);
1060         return -1;
1061 }
1062
1063 /*
1064  * Test lookup perf for multi-writer:
1065  * Reader(s) lookup keys present in the table and likely on the shift-path while
1066  * Writers add keys causing key-shiftsi.
1067  * Writers are running in parallel, on different data plane cores.
1068  */
1069 static int
1070 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1071                            int htm, int ext_bkt)
1072 {
1073         unsigned int n, m, k;
1074         uint64_t i;
1075         int use_jhash = 0;
1076         uint8_t write_type;
1077         uint8_t read_type = READ_PASS_SHIFT_PATH;
1078
1079         rte_atomic64_init(&greads);
1080         rte_atomic64_init(&gread_cycles);
1081
1082         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1083                 goto err;
1084         printf("\nTest: Multi-add-lookup\n");
1085         uint8_t pos_core;
1086         for (m = 1; m < NUM_TEST; m++) {
1087                 /* Calculate keys added by each writer */
1088                 tbl_rwc_test_param.single_insert =
1089                         tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1090                 for (k = 0; k < 2; k++) {
1091                         if (k == 1) {
1092                                 printf("\n** With bulk-lookup **\n");
1093                                 read_type |= BULK_LOOKUP;
1094                         }
1095                         for (n = 0; n < NUM_TEST; n++) {
1096                                 unsigned int tot_lcore  = rte_lcore_count();
1097                                 if (tot_lcore < (rwc_core_cnt[n] +
1098                                      rwc_core_cnt[m] + 1))
1099                                         goto finish;
1100
1101                                 printf("\nNumber of writers: %u",
1102                                        rwc_core_cnt[m]);
1103                                 printf("\nNumber of readers: %u\n",
1104                                        rwc_core_cnt[n]);
1105
1106                                 rte_atomic64_clear(&greads);
1107                                 rte_atomic64_clear(&gread_cycles);
1108
1109                                 rte_hash_reset(tbl_rwc_test_param.h);
1110                                 writer_done = 0;
1111                                 for (i = 0; i < 4; i++)
1112                                         multi_writer_done[i] = 0;
1113                                 write_type = WRITE_NO_KEY_SHIFT;
1114                                 if (write_keys(write_type) < 0)
1115                                         goto err;
1116
1117                                 /* Launch reader(s) */
1118                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1119                                         rte_eal_remote_launch(test_rwc_reader,
1120                                                 (void *)(uintptr_t)read_type,
1121                                                 enabled_core_ids[i]);
1122                                 write_type = WRITE_KEY_SHIFT;
1123                                 pos_core = 0;
1124
1125                                 /* Launch writers */
1126                                 for (; i <= rwc_core_cnt[m]
1127                                      + rwc_core_cnt[n]; i++) {
1128                                         rte_eal_remote_launch
1129                                                 (test_rwc_multi_writer,
1130                                                 (void *)(uintptr_t)pos_core,
1131                                                 enabled_core_ids[i]);
1132                                         pos_core++;
1133                                 }
1134
1135                                 /* Wait for writers to complete */
1136                                 for (i = 0; i < rwc_core_cnt[m]; i++)
1137                                         while
1138                                                 (multi_writer_done[i] == 0);
1139                                 writer_done = 1;
1140
1141                                 rte_eal_mp_wait_lcore();
1142
1143                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1144                                         if (lcore_config[i].ret < 0)
1145                                                 goto err;
1146
1147                                 unsigned long long cycles_per_lookup =
1148                                         rte_atomic64_read(&gread_cycles)
1149                                         / rte_atomic64_read(&greads);
1150                                 rwc_perf_results->multi_rw[m][k][n]
1151                                         = cycles_per_lookup;
1152                                 printf("Cycles per lookup: %llu\n",
1153                                        cycles_per_lookup);
1154                         }
1155                 }
1156         }
1157
1158 finish:
1159         rte_hash_free(tbl_rwc_test_param.h);
1160         return 0;
1161
1162 err:
1163         rte_hash_free(tbl_rwc_test_param.h);
1164         return -1;
1165 }
1166
1167 /*
1168  * Test lookup perf:
1169  * Reader(s) lookup keys present in the extendable bkt.
1170  */
1171 static int
1172 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1173                                 int rwc_lf, int htm, int ext_bkt)
1174 {
1175         unsigned int n, m;
1176         uint64_t i;
1177         int use_jhash = 0;
1178         uint8_t write_type;
1179         uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1180
1181         rte_atomic64_init(&greads);
1182         rte_atomic64_init(&gread_cycles);
1183
1184         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1185                 goto err;
1186         printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1187         for (m = 0; m < 2; m++) {
1188                 if (m == 1) {
1189                         printf("\n** With bulk-lookup **\n");
1190                         read_type |= BULK_LOOKUP;
1191                 }
1192                 for (n = 0; n < NUM_TEST; n++) {
1193                         unsigned int tot_lcore = rte_lcore_count();
1194                         if (tot_lcore < rwc_core_cnt[n] + 1)
1195                                 goto finish;
1196
1197                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1198
1199                         rte_atomic64_clear(&greads);
1200                         rte_atomic64_clear(&gread_cycles);
1201
1202                         rte_hash_reset(tbl_rwc_test_param.h);
1203                         write_type = WRITE_NO_KEY_SHIFT;
1204                         if (write_keys(write_type) < 0)
1205                                 goto err;
1206                         write_type = WRITE_KEY_SHIFT;
1207                         if (write_keys(write_type) < 0)
1208                                 goto err;
1209                         writer_done = 0;
1210                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1211                                 rte_eal_remote_launch(test_rwc_reader,
1212                                                 (void *)(uintptr_t)read_type,
1213                                                         enabled_core_ids[i]);
1214                         for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1215                              i++) {
1216                                 if (rte_hash_del_key(tbl_rwc_test_param.h,
1217                                         tbl_rwc_test_param.keys_ks_extbkt + i)
1218                                                         < 0) {
1219                                         printf("Delete Failed: %u\n",
1220                                         tbl_rwc_test_param.keys_ks_extbkt[i]);
1221                                         goto err;
1222                                 }
1223                         }
1224                         writer_done = 1;
1225                         rte_eal_mp_wait_lcore();
1226
1227                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1228                                 if (lcore_config[i].ret < 0)
1229                                         goto err;
1230
1231                         unsigned long long cycles_per_lookup =
1232                                 rte_atomic64_read(&gread_cycles) /
1233                                 rte_atomic64_read(&greads);
1234                         rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1235                                                 = cycles_per_lookup;
1236                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1237                 }
1238         }
1239
1240 finish:
1241         rte_hash_free(tbl_rwc_test_param.h);
1242         return 0;
1243
1244 err:
1245         rte_hash_free(tbl_rwc_test_param.h);
1246         return -1;
1247 }
1248
1249 static int
1250 test_hash_readwrite_lf_main(void)
1251 {
1252         /*
1253          * Variables used to choose different tests.
1254          * rwc_lf indicates if read-write concurrency lock-free support is
1255          * enabled.
1256          * htm indicates if Hardware transactional memory support is enabled.
1257          */
1258         int rwc_lf = 0;
1259         int htm;
1260         int use_jhash = 0;
1261         int ext_bkt = 0;
1262         if (rte_lcore_count() == 1) {
1263                 printf("More than one lcore is required "
1264                         "to do read write lock-free concurrency test\n");
1265                 return -1;
1266         }
1267
1268         setlocale(LC_NUMERIC, "");
1269
1270         if (rte_tm_supported())
1271                 htm = 1;
1272         else
1273                 htm = 0;
1274
1275         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1276                 return -1;
1277         if (generate_keys() != 0)
1278                 return -1;
1279         if (get_enabled_cores_list() != 0)
1280                 return -1;
1281
1282         if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1283                 rwc_lf = 1;
1284                 ext_bkt = 1;
1285                 printf("Test lookup with read-write concurrency lock free support"
1286                        " enabled\n");
1287                 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1288                                                         htm, ext_bkt) < 0)
1289                         return -1;
1290                 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1291                                                         htm, ext_bkt) < 0)
1292                         return -1;
1293                 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1294                                                         htm, ext_bkt) < 0)
1295                         return -1;
1296                 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1297                                                         htm, ext_bkt) < 0)
1298                         return -1;
1299                 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1300                                                  ext_bkt) < 0)
1301                         return -1;
1302                 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1303                                                ext_bkt) < 0)
1304                         return -1;
1305                 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1306                                                         htm, ext_bkt) < 0)
1307                         return -1;
1308         }
1309         printf("\nTest lookup with read-write concurrency lock free support"
1310                " disabled\n");
1311         rwc_lf = 0;
1312         if (!htm) {
1313                 printf("With HTM Disabled\n");
1314                 if (!RUN_WITH_HTM_DISABLED) {
1315                         printf("Enable RUN_WITH_HTM_DISABLED to test with"
1316                                " lock-free disabled");
1317                         goto results;
1318                 }
1319         } else
1320                 printf("With HTM Enabled\n");
1321         if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1322                                            ext_bkt) < 0)
1323                 return -1;
1324         if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1325                                                 ext_bkt) < 0)
1326                 return -1;
1327         if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1328                                                 htm, ext_bkt) < 0)
1329                 return -1;
1330         if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1331                                                 ext_bkt) < 0)
1332                 return -1;
1333         if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1334                                          ext_bkt) < 0)
1335                 return -1;
1336         if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1337                                                         ext_bkt) < 0)
1338                 return -1;
1339         if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1340                                                 htm, ext_bkt) < 0)
1341                 return -1;
1342 results:
1343         printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1344         int i, j, k;
1345         for (j = 0; j < 2; j++) {
1346                 if (j == 1)
1347                         printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1348                                "**********#######\n\n");
1349                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1350                         "\t\t\t\t_________________\n");
1351                 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1352                        "\t\t\tCycles per lookup\n");
1353                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1354                        "\t\t\t_________________\n");
1355                 for (i = 0; i < NUM_TEST; i++) {
1356                         printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1357                         printf("Enabled\t\t");
1358                         printf("N/A\t\t");
1359                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1360                                 "%u\n\t\t\t\t\t\t\t\t",
1361                                 rwc_lf_results.w_no_ks_r_hit[j][i]);
1362                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1363                                 "%u\n\t\t\t\t\t\t\t\t",
1364                                 rwc_lf_results.w_no_ks_r_miss[j][i]);
1365                         printf("Hash add - key-shifts, lookup - hit"
1366                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1367                                rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1368                         printf("Hash add - key-shifts, lookup - hit "
1369                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1370                                rwc_lf_results.w_ks_r_hit_sp[j][i]);
1371                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1372                                 "%u\n\t\t\t\t\t\t\t\t",
1373                                 rwc_lf_results.w_ks_r_miss[j][i]);
1374                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1375                                 "%u\n\n\t\t\t\t",
1376                                 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1377
1378                         printf("Disabled\t");
1379                         if (htm)
1380                                 printf("Enabled\t\t");
1381                         else
1382                                 printf("Disabled\t");
1383                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1384                                 "%u\n\t\t\t\t\t\t\t\t",
1385                                 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1386                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1387                                 "%u\n\t\t\t\t\t\t\t\t",
1388                                 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1389                         printf("Hash add - key-shifts, lookup - hit "
1390                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1391                                rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1392                         printf("Hash add - key-shifts, lookup - hit "
1393                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1394                                rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1395                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1396                                "%u\n\t\t\t\t\t\t\t\t",
1397                                rwc_non_lf_results.w_ks_r_miss[j][i]);
1398                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1399                                 "%u\n",
1400                                 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1401
1402                         printf("_______\t\t_______\t\t_________\t___\t\t"
1403                                "_________\t\t\t\t\t\t_________________\n");
1404                 }
1405
1406                 for (i = 1; i < NUM_TEST; i++) {
1407                         for (k = 0; k < NUM_TEST; k++) {
1408                                 printf("%u", rwc_core_cnt[i]);
1409                                 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1410                                 printf("Enabled\t\t");
1411                                 printf("N/A\t\t");
1412                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1413                                        "\t\t",
1414                                        rwc_lf_results.multi_rw[i][j][k]);
1415                                 printf("Disabled\t");
1416                                 if (htm)
1417                                         printf("Enabled\t\t");
1418                                 else
1419                                         printf("Disabled\t");
1420                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1421                                        rwc_non_lf_results.multi_rw[i][j][k]);
1422
1423                                 printf("_______\t\t_______\t\t_________\t___"
1424                                        "\t\t_________\t\t\t\t\t\t"
1425                                        "_________________\n");
1426                         }
1427                 }
1428         }
1429         rte_free(tbl_rwc_test_param.keys);
1430         rte_free(tbl_rwc_test_param.keys_no_ks);
1431         rte_free(tbl_rwc_test_param.keys_ks);
1432         rte_free(tbl_rwc_test_param.keys_absent);
1433         rte_free(tbl_rwc_test_param.keys_shift_path);
1434         rte_free(scanned_bkts);
1435         return 0;
1436 }
1437
1438 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);