app/test/test_hash_readwrite_lf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Arm Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
  20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
  21 #endif
  22
  23 #define BULK_LOOKUP_SIZE 32
  24
  25 #define RUN_WITH_HTM_DISABLED 0
  26
  27 #if (RUN_WITH_HTM_DISABLED)
  28
  29 #define TOTAL_ENTRY (5*1024)
  30 #define TOTAL_INSERT (5*1024)
  31
  32 #else
  33
  34 #define TOTAL_ENTRY (4*1024*1024)
  35 #define TOTAL_INSERT (4*1024*1024)
  36
  37 #endif
  38
  39 #define READ_FAIL 1
  40 #define READ_PASS_NO_KEY_SHIFTS 2
  41 #define READ_PASS_SHIFT_PATH 4
  42 #define READ_PASS_NON_SHIFT_PATH 8
  43 #define BULK_LOOKUP 16
  44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
  45
  46 #define WRITE_NO_KEY_SHIFT 0
  47 #define WRITE_KEY_SHIFT 1
  48 #define WRITE_EXT_BKT 2
  49
  50 #define NUM_TEST 3
  51 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
  52
  53 struct rwc_perf {
  54         uint32_t w_no_ks_r_hit[2][NUM_TEST];
  55         uint32_t w_no_ks_r_miss[2][NUM_TEST];
  56         uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
  57         uint32_t w_ks_r_hit_sp[2][NUM_TEST];
  58         uint32_t w_ks_r_miss[2][NUM_TEST];
  59         uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
  60         uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
  61 };
  62
  63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
  64
  65 struct {
  66         uint32_t *keys;
  67         uint32_t *keys_no_ks;
  68         uint32_t *keys_ks;
  69         uint32_t *keys_absent;
  70         uint32_t *keys_shift_path;
  71         uint32_t *keys_non_shift_path;
  72         uint32_t *keys_ext_bkt;
  73         uint32_t *keys_ks_extbkt;
  74         uint32_t count_keys_no_ks;
  75         uint32_t count_keys_ks;
  76         uint32_t count_keys_absent;
  77         uint32_t count_keys_shift_path;
  78         uint32_t count_keys_non_shift_path;
  79         uint32_t count_keys_extbkt;
  80         uint32_t count_keys_ks_extbkt;
  81         uint32_t single_insert;
  82         struct rte_hash *h;
  83 } tbl_rwc_test_param;
  84
  85 static rte_atomic64_t gread_cycles;
  86 static rte_atomic64_t greads;
  87
  88 static volatile uint8_t writer_done;
  89
  90 uint16_t enabled_core_ids[RTE_MAX_LCORE];
  91
  92 uint8_t *scanned_bkts;
  93
  94 static inline uint16_t
  95 get_short_sig(const hash_sig_t hash)
  96 {
  97         return hash >> 16;
  98 }
  99
 100 static inline uint32_t
 101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 102                       const hash_sig_t hash)
 103 {
 104         uint32_t num_buckets;
 105         uint32_t bucket_bitmask;
 106         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 107         bucket_bitmask = num_buckets - 1;
 108         return hash & bucket_bitmask;
 109 }
 110
 111 static inline uint32_t
 112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 113                         uint32_t cur_bkt_idx, uint16_t sig)
 114 {
 115         uint32_t num_buckets;
 116         uint32_t bucket_bitmask;
 117         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 118         bucket_bitmask = num_buckets - 1;
 119         return (cur_bkt_idx ^ sig) & bucket_bitmask;
 120 }
 121
 122
 123 static inline int
 124 get_enabled_cores_list(void)
 125 {
 126         uint32_t i = 0;
 127         uint16_t core_id;
 128         uint32_t max_cores = rte_lcore_count();
 129         RTE_LCORE_FOREACH(core_id) {
 130                 enabled_core_ids[i] = core_id;
 131                 i++;
 132         }
 133
 134         if (i != max_cores) {
 135                 printf("Number of enabled cores in list is different from "
 136                         "number given by rte_lcore_count()\n");
 137                 return -1;
 138         }
 139         return 0;
 140 }
 141
 142 static int
 143 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
 144 {
 145         struct rte_hash *handle;
 146
 147         struct rte_hash_parameters hash_params = {
 148                 .entries = TOTAL_ENTRY,
 149                 .key_len = sizeof(uint32_t),
 150                 .hash_func_init_val = 0,
 151                 .socket_id = rte_socket_id(),
 152         };
 153
 154         if (use_jhash)
 155                 hash_params.hash_func = rte_jhash;
 156         else
 157                 hash_params.hash_func = rte_hash_crc;
 158
 159         if (rwc_lf)
 160                 hash_params.extra_flag =
 161                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
 162                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 163         else if (htm)
 164                 hash_params.extra_flag =
 165                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 166                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 167                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 168         else
 169                 hash_params.extra_flag =
 170                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 171                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 172
 173         if (ext_bkt)
 174                 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
 175
 176         hash_params.name = "tests";
 177
 178         handle = rte_hash_create(&hash_params);
 179         if (handle == NULL) {
 180                 printf("hash creation failed");
 181                 return -1;
 182         }
 183
 184         tbl_rwc_test_param.h = handle;
 185         return 0;
 186 }
 187
 188 static inline int
 189 check_bucket(uint32_t bkt_idx, uint32_t key)
 190 {
 191         uint32_t iter;
 192         uint32_t prev_iter;
 193         uint32_t diff;
 194         uint32_t count = 0;
 195         const void *next_key;
 196         void *next_data;
 197
 198         /* Temporary bucket to hold the keys */
 199         uint32_t keys_in_bkt[8];
 200
 201         iter = bkt_idx * 8;
 202         prev_iter = iter;
 203         while (rte_hash_iterate(tbl_rwc_test_param.h,
 204                         &next_key, &next_data, &iter) >= 0) {
 205
 206                 /* Check for duplicate entries */
 207                 if (*(const uint32_t *)next_key == key)
 208                         return 1;
 209
 210                 /* Identify if there is any free entry in the bucket */
 211                 diff = iter - prev_iter;
 212                 if (diff > 1)
 213                         break;
 214
 215                 prev_iter = iter;
 216                 keys_in_bkt[count] = *(const uint32_t *)next_key;
 217                 count++;
 218
 219                 /* All entries in the bucket are occupied */
 220                 if (count == 8) {
 221
 222                         /*
 223                          * Check if bucket was not scanned before, to avoid
 224                          * duplicate keys.
 225                          */
 226                         if (scanned_bkts[bkt_idx] == 0) {
 227                                 /*
 228                                  * Since this bucket (pointed to by bkt_idx) is
 229                                  * full, it is likely that key(s) in this
 230                                  * bucket will be on the shift path, when
 231                                  * collision occurs. Thus, add it to
 232                                  * keys_shift_path.
 233                                  */
 234                                 memcpy(tbl_rwc_test_param.keys_shift_path +
 235                                         tbl_rwc_test_param.count_keys_shift_path
 236                                         , keys_in_bkt, 32);
 237                                 tbl_rwc_test_param.count_keys_shift_path += 8;
 238                                 scanned_bkts[bkt_idx] = 1;
 239                         }
 240                         return -1;
 241                 }
 242         }
 243         return 0;
 244 }
 245
 246 static int
 247 generate_keys(void)
 248 {
 249         uint32_t *keys = NULL;
 250         uint32_t *keys_no_ks = NULL;
 251         uint32_t *keys_ks = NULL;
 252         uint32_t *keys_absent = NULL;
 253         uint32_t *keys_non_shift_path = NULL;
 254         uint32_t *keys_ext_bkt = NULL;
 255         uint32_t *keys_ks_extbkt = NULL;
 256         uint32_t *found = NULL;
 257         uint32_t count_keys_no_ks = 0;
 258         uint32_t count_keys_ks = 0;
 259         uint32_t count_keys_extbkt = 0;
 260         uint32_t i;
 261
 262         if (init_params(0, 0, 0, 0) != 0)
 263                 return -1;
 264
 265         /*
 266          * keys will consist of a) keys whose addition to the hash table
 267          * will result in shifting of the existing keys to their alternate
 268          * locations b) keys whose addition to the hash table will not result
 269          * in shifting of the existing keys.
 270          */
 271         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 272         if (keys == NULL) {
 273                 printf("RTE_MALLOC failed\n");
 274                 goto err;
 275         }
 276
 277         /*
 278          * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
 279          * will NOT result in shifting of the existing keys to their alternate
 280          * locations. Roughly around 900K keys.
 281          */
 282         keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 283         if (keys_no_ks == NULL) {
 284                 printf("RTE_MALLOC failed\n");
 285                 goto err;
 286         }
 287
 288         /*
 289          * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
 290          * result in shifting of the existing keys to their alternate locations.
 291          * Roughly around 146K keys. There might be repeating keys. More code is
 292          * required to filter out these keys which will complicate the test case
 293          */
 294         keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 295         if (keys_ks == NULL) {
 296                 printf("RTE_MALLOC failed\n");
 297                 goto err;
 298         }
 299
 300         /* Used to identify keys not inserted in the hash table */
 301         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 302         if (found == NULL) {
 303                 printf("RTE_MALLOC failed\n");
 304                 goto err;
 305         }
 306
 307         /*
 308          * This consist of keys not inserted to the hash table.
 309          * Used to test perf of lookup on keys that do not exist in the table.
 310          */
 311         keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 312         if (keys_absent == NULL) {
 313                 printf("RTE_MALLOC failed\n");
 314                 goto err;
 315         }
 316
 317         /*
 318          * This consist of keys which are likely to be on the shift
 319          * path (i.e. being moved to alternate location), when collision occurs
 320          * on addition of a key to an already full primary bucket.
 321          * Used to test perf of lookup on keys that are on the shift path.
 322          */
 323         tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
 324                                                         TOTAL_INSERT, 0);
 325         if (tbl_rwc_test_param.keys_shift_path == NULL) {
 326                 printf("RTE_MALLOC failed\n");
 327                 goto err;
 328         }
 329
 330         /*
 331          * This consist of keys which are never on the shift
 332          * path (i.e. being moved to alternate location), when collision occurs
 333          * on addition of a key to an already full primary bucket.
 334          * Used to test perf of lookup on keys that are not on the shift path.
 335          */
 336         keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
 337                                          0);
 338         if (keys_non_shift_path == NULL) {
 339                 printf("RTE_MALLOC failed\n");
 340                 goto err;
 341         }
 342
 343         /*
 344          * This consist of keys which will be stored in extended buckets
 345          */
 346         keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 347         if (keys_ext_bkt == NULL) {
 348                 printf("RTE_MALLOC failed\n");
 349                 goto err;
 350         }
 351
 352         /*
 353          * This consist of keys which when deleted causes shifting of keys
 354          * in extended buckets to respective secondary buckets
 355          */
 356         keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 357         if (keys_ks_extbkt == NULL) {
 358                 printf("RTE_MALLOC failed\n");
 359                 goto err;
 360         }
 361
 362         hash_sig_t sig;
 363         uint32_t prim_bucket_idx;
 364         uint32_t sec_bucket_idx;
 365         uint16_t short_sig;
 366         uint32_t num_buckets;
 367         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 368         int ret;
 369
 370         /*
 371          * Used to mark bkts in which at least one key was shifted to its
 372          * alternate location
 373          */
 374         scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
 375         if (scanned_bkts == NULL) {
 376                 printf("RTE_MALLOC failed\n");
 377                 goto err;
 378         }
 379
 380         tbl_rwc_test_param.keys = keys;
 381         tbl_rwc_test_param.keys_no_ks = keys_no_ks;
 382         tbl_rwc_test_param.keys_ks = keys_ks;
 383         tbl_rwc_test_param.keys_absent = keys_absent;
 384         tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
 385         tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
 386         tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
 387         /* Generate keys by adding previous two keys, neglect overflow */
 388         printf("Generating keys...\n");
 389         keys[0] = 0;
 390         keys[1] = 1;
 391         for (i = 2; i < TOTAL_INSERT; i++)
 392                 keys[i] = keys[i-1] + keys[i-2];
 393
 394         /* Segregate keys into keys_no_ks and keys_ks */
 395         for (i = 0; i < TOTAL_INSERT; i++) {
 396                 /* Check if primary bucket has space.*/
 397                 sig = rte_hash_hash(tbl_rwc_test_param.h,
 398                                         tbl_rwc_test_param.keys+i);
 399                 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
 400                                                         sig);
 401                 ret = check_bucket(prim_bucket_idx, keys[i]);
 402                 if (ret < 0) {
 403                         /*
 404                          * Primary bucket is full, this key will result in
 405                          * shifting of the keys to their alternate locations.
 406                          */
 407                         keys_ks[count_keys_ks] = keys[i];
 408                         count_keys_ks++;
 409                 } else if (ret == 0) {
 410                         /*
 411                          * Primary bucket has space, this key will not result in
 412                          * shifting of the keys. Hence, add key to the table.
 413                          */
 414                         ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
 415                                                         keys+i,
 416                                                         (void *)((uintptr_t)i));
 417                         if (ret < 0) {
 418                                 printf("writer failed %"PRIu32"\n", i);
 419                                 break;
 420                         }
 421                         keys_no_ks[count_keys_no_ks] = keys[i];
 422                         count_keys_no_ks++;
 423                 }
 424         }
 425
 426         for (i = 0; i < count_keys_no_ks; i++) {
 427                 /*
 428                  * Identify keys in keys_no_ks with value less than
 429                  * 4M (HTM enabled) OR 5K (HTM disabled)
 430                  */
 431                 if (keys_no_ks[i] < TOTAL_INSERT)
 432                         found[keys_no_ks[i]]++;
 433         }
 434
 435         for (i = 0; i < count_keys_ks; i++) {
 436                 /*
 437                  * Identify keys in keys_ks with value less than
 438                  * 4M (HTM enabled) OR 5K (HTM disabled)
 439                  */
 440                 if (keys_ks[i] < TOTAL_INSERT)
 441                         found[keys_ks[i]]++;
 442         }
 443
 444         uint32_t count_keys_absent = 0;
 445         for (i = 0; i < TOTAL_INSERT; i++) {
 446                 /*
 447                  * Identify missing keys between 0 and
 448                  * 4M (HTM enabled) OR 5K (HTM disabled)
 449                  */
 450                 if (found[i] == 0)
 451                         keys_absent[count_keys_absent++] = i;
 452         }
 453
 454         /* Find keys that will not be on the shift path */
 455         uint32_t iter;
 456         const void *next_key;
 457         void *next_data;
 458         uint32_t count = 0;
 459         for (i = 0; i < num_buckets; i++) {
 460                 /* Check bucket for no keys shifted to alternate locations */
 461                 if (scanned_bkts[i] == 0) {
 462                         iter = i * 8;
 463                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 464                                 &next_key, &next_data, &iter) >= 0) {
 465
 466                                 /* Check if key belongs to the current bucket */
 467                                 if (i >= (iter-1)/8)
 468                                         keys_non_shift_path[count++]
 469                                                 = *(const uint32_t *)next_key;
 470                                 else
 471                                         break;
 472                         }
 473                 }
 474         }
 475
 476         tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
 477         tbl_rwc_test_param.count_keys_ks = count_keys_ks;
 478         tbl_rwc_test_param.count_keys_absent = count_keys_absent;
 479         tbl_rwc_test_param.count_keys_non_shift_path = count;
 480
 481         memset(scanned_bkts, 0, num_buckets);
 482         count = 0;
 483         /* Find keys that will be in extended buckets */
 484         for (i = 0; i < count_keys_ks; i++) {
 485                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
 486                 if (ret < 0) {
 487                         /* Key will be added to ext bkt */
 488                         keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
 489                         /* Sec bkt to be added to keys_ks_extbkt */
 490                         sig = rte_hash_hash(tbl_rwc_test_param.h,
 491                                         tbl_rwc_test_param.keys_ks + i);
 492                         prim_bucket_idx = get_prim_bucket_index(
 493                                                 tbl_rwc_test_param.h, sig);
 494                         short_sig = get_short_sig(sig);
 495                         sec_bucket_idx = get_alt_bucket_index(
 496                                                 tbl_rwc_test_param.h,
 497                                                 prim_bucket_idx, short_sig);
 498                         if (scanned_bkts[sec_bucket_idx] == 0)
 499                                 scanned_bkts[sec_bucket_idx] = 1;
 500                 }
 501         }
 502
 503         /* Find keys that will shift keys in ext bucket*/
 504         for (i = 0; i < num_buckets; i++) {
 505                 if (scanned_bkts[i] == 1) {
 506                         iter = i * 8;
 507                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 508                                 &next_key, &next_data, &iter) >= 0) {
 509                                 /* Check if key belongs to the current bucket */
 510                                 if (i >= (iter-1)/8)
 511                                         keys_ks_extbkt[count++]
 512                                                 = *(const uint32_t *)next_key;
 513                                 else
 514                                         break;
 515                         }
 516                 }
 517         }
 518
 519         tbl_rwc_test_param.count_keys_ks_extbkt = count;
 520         tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
 521
 522         printf("\nCount of keys NOT causing shifting of existing keys to "
 523         "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
 524         printf("\nCount of keys causing shifting of existing keys to alternate "
 525                 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
 526         printf("Count of absent keys that will never be added to the hash "
 527                 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
 528         printf("Count of keys likely to be on the shift path: %d\n\n",
 529                tbl_rwc_test_param.count_keys_shift_path);
 530         printf("Count of keys not likely to be on the shift path: %d\n\n",
 531                tbl_rwc_test_param.count_keys_non_shift_path);
 532         printf("Count of keys in extended buckets: %d\n\n",
 533                tbl_rwc_test_param.count_keys_extbkt);
 534         printf("Count of keys shifting keys in ext buckets: %d\n\n",
 535                tbl_rwc_test_param.count_keys_ks_extbkt);
 536
 537         rte_free(found);
 538         rte_free(scanned_bkts);
 539         rte_hash_free(tbl_rwc_test_param.h);
 540         return 0;
 541
 542 err:
 543         rte_free(keys);
 544         rte_free(keys_no_ks);
 545         rte_free(keys_ks);
 546         rte_free(keys_absent);
 547         rte_free(found);
 548         rte_free(tbl_rwc_test_param.keys_shift_path);
 549         rte_free(keys_non_shift_path);
 550         rte_free(keys_ext_bkt);
 551         rte_free(keys_ks_extbkt);
 552         rte_free(scanned_bkts);
 553         rte_hash_free(tbl_rwc_test_param.h);
 554         return -1;
 555 }
 556
 557 static int
 558 test_rwc_reader(__attribute__((unused)) void *arg)
 559 {
 560         uint32_t i, j;
 561         int ret;
 562         uint64_t begin, cycles;
 563         uint32_t loop_cnt = 0;
 564         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 565         uint32_t read_cnt;
 566         uint32_t *keys;
 567         uint32_t extra_keys;
 568         int32_t *pos;
 569         void *temp_a[BULK_LOOKUP_SIZE];
 570
 571         /* Used to identify keys not inserted in the hash table */
 572         pos = rte_malloc(NULL, sizeof(uint32_t) * BULK_LOOKUP_SIZE, 0);
 573         if (pos == NULL) {
 574                 printf("RTE_MALLOC failed\n");
 575                 return -1;
 576         }
 577
 578         if (read_type & READ_FAIL) {
 579                 keys = tbl_rwc_test_param.keys_absent;
 580                 read_cnt = tbl_rwc_test_param.count_keys_absent;
 581         } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
 582                 keys = tbl_rwc_test_param.keys_no_ks;
 583                 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
 584         } else if (read_type & READ_PASS_SHIFT_PATH) {
 585                 keys = tbl_rwc_test_param.keys_shift_path;
 586                 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
 587         } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
 588                 keys = tbl_rwc_test_param.keys_ext_bkt;
 589                 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
 590         } else {
 591                 keys = tbl_rwc_test_param.keys_non_shift_path;
 592                 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
 593         }
 594
 595         extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
 596
 597         begin = rte_rdtsc_precise();
 598         do {
 599                 if (read_type & BULK_LOOKUP) {
 600                         for (i = 0; i < (read_cnt - extra_keys);
 601                              i += BULK_LOOKUP_SIZE) {
 602                                 /* Array of  pointer to the list of keys */
 603                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 604                                         temp_a[j] = keys + i + j;
 605                                 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 606                                                    (const void **)
 607                                                    ((uintptr_t)temp_a),
 608                                                    BULK_LOOKUP_SIZE, pos);
 609                                 /* Validate lookup result */
 610                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 611                                         if ((read_type & READ_FAIL &&
 612                                              pos[j] != -ENOENT) ||
 613                                             (!(read_type & READ_FAIL) &&
 614                                              pos[j] == -ENOENT)) {
 615                                                 printf("lookup failed!"
 616                                                        "%"PRIu32"\n",
 617                                                        keys[i + j]);
 618                                                 return -1;
 619                                         }
 620                         }
 621                         for (j = 0; j < extra_keys; j++)
 622                                 temp_a[j] = keys + i + j;
 623
 624                         rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 625                                            (const void **)
 626                                            ((uintptr_t)temp_a),
 627                                            extra_keys, pos);
 628                         for (j = 0; j < extra_keys; j++)
 629                                 if ((read_type & READ_FAIL &&
 630                                      pos[j] != -ENOENT) ||
 631                                     (!(read_type & READ_FAIL) &&
 632                                      pos[j] == -ENOENT)) {
 633                                         printf("lookup failed! %"PRIu32"\n",
 634                                                keys[i + j]);
 635                                         return -1;
 636                                 }
 637                 } else {
 638                         for (i = 0; i < read_cnt; i++) {
 639                                 ret = rte_hash_lookup
 640                                         (tbl_rwc_test_param.h, keys + i);
 641                                 if (((read_type & READ_FAIL) &&
 642                                      (ret != -ENOENT)) ||
 643                                     (!(read_type & READ_FAIL) &&
 644                                         ret == -ENOENT)) {
 645                                         printf("lookup failed! %"PRIu32"\n",
 646                                                keys[i]);
 647                                         return -1;
 648                                 }
 649                         }
 650                 }
 651                 loop_cnt++;
 652         } while (!writer_done);
 653
 654         cycles = rte_rdtsc_precise() - begin;
 655         rte_atomic64_add(&gread_cycles, cycles);
 656         rte_atomic64_add(&greads, read_cnt*loop_cnt);
 657         return 0;
 658 }
 659
 660 static int
 661 write_keys(uint8_t write_type)
 662 {
 663         uint32_t i;
 664         int ret;
 665         uint32_t key_cnt = 0;
 666         uint32_t *keys;
 667         if (write_type == WRITE_KEY_SHIFT) {
 668                 key_cnt = tbl_rwc_test_param.count_keys_ks;
 669                 keys = tbl_rwc_test_param.keys_ks;
 670         } else if (write_type == WRITE_NO_KEY_SHIFT) {
 671                 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
 672                 keys = tbl_rwc_test_param.keys_no_ks;
 673         } else if (write_type == WRITE_EXT_BKT) {
 674                 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
 675                 keys = tbl_rwc_test_param.keys_ext_bkt;
 676         }
 677         for (i = 0; i < key_cnt; i++) {
 678                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
 679                 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
 680                         printf("writer failed %"PRIu32"\n", i);
 681                         return -1;
 682                 }
 683         }
 684         return 0;
 685 }
 686
 687 static int
 688 test_rwc_multi_writer(__attribute__((unused)) void *arg)
 689 {
 690         uint32_t i, offset;
 691         uint32_t pos_core = (uint32_t)((uintptr_t)arg);
 692         offset = pos_core * tbl_rwc_test_param.single_insert;
 693         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
 694                 rte_hash_add_key(tbl_rwc_test_param.h,
 695                                  tbl_rwc_test_param.keys_ks + i);
 696         return 0;
 697 }
 698
 699 /*
 700  * Test lookup perf:
 701  * Reader(s) lookup keys present in the table.
 702  */
 703 static int
 704 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 705                                 int htm, int ext_bkt)
 706 {
 707         unsigned int n, m;
 708         uint64_t i;
 709         int use_jhash = 0;
 710         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 711         uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
 712
 713         rte_atomic64_init(&greads);
 714         rte_atomic64_init(&gread_cycles);
 715
 716         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 717                 goto err;
 718         printf("\nTest: Hash add - no key-shifts, read - hit\n");
 719         for (m = 0; m < 2; m++) {
 720                 if (m == 1) {
 721                         printf("\n** With bulk-lookup **\n");
 722                         read_type |= BULK_LOOKUP;
 723                 }
 724                 for (n = 0; n < NUM_TEST; n++) {
 725                         unsigned int tot_lcore = rte_lcore_count();
 726                         if (tot_lcore < rwc_core_cnt[n] + 1)
 727                                 goto finish;
 728
 729                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 730
 731                         rte_atomic64_clear(&greads);
 732                         rte_atomic64_clear(&gread_cycles);
 733
 734                         rte_hash_reset(tbl_rwc_test_param.h);
 735                         writer_done = 0;
 736                         if (write_keys(write_type) < 0)
 737                                 goto err;
 738                         writer_done = 1;
 739                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 740                                 rte_eal_remote_launch(test_rwc_reader,
 741                                                 (void *)(uintptr_t)read_type,
 742                                                         enabled_core_ids[i]);
 743
 744                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 745                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 746                                         goto err;
 747
 748                         unsigned long long cycles_per_lookup =
 749                                 rte_atomic64_read(&gread_cycles) /
 750                                 rte_atomic64_read(&greads);
 751                         rwc_perf_results->w_no_ks_r_hit[m][n]
 752                                                 = cycles_per_lookup;
 753                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 754                 }
 755         }
 756
 757 finish:
 758         rte_hash_free(tbl_rwc_test_param.h);
 759         return 0;
 760
 761 err:
 762         rte_eal_mp_wait_lcore();
 763         rte_hash_free(tbl_rwc_test_param.h);
 764         return -1;
 765 }
 766
 767 /*
 768  * Test lookup perf:
 769  * Reader(s) lookup keys absent in the table while
 770  * 'Main' thread adds with no key-shifts.
 771  */
 772 static int
 773 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 774                                 int htm, int ext_bkt)
 775 {
 776         unsigned int n, m;
 777         uint64_t i;
 778         int use_jhash = 0;
 779         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 780         uint8_t read_type = READ_FAIL;
 781         int ret;
 782
 783         rte_atomic64_init(&greads);
 784         rte_atomic64_init(&gread_cycles);
 785
 786         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 787                 goto err;
 788         printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
 789         for (m = 0; m < 2; m++) {
 790                 if (m == 1) {
 791                         printf("\n** With bulk-lookup **\n");
 792                         read_type |= BULK_LOOKUP;
 793                 }
 794                 for (n = 0; n < NUM_TEST; n++) {
 795                         unsigned int tot_lcore = rte_lcore_count();
 796                         if (tot_lcore < rwc_core_cnt[n] + 1)
 797                                 goto finish;
 798
 799                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 800
 801                         rte_atomic64_clear(&greads);
 802                         rte_atomic64_clear(&gread_cycles);
 803
 804                         rte_hash_reset(tbl_rwc_test_param.h);
 805                         writer_done = 0;
 806
 807                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 808                                 rte_eal_remote_launch(test_rwc_reader,
 809                                                 (void *)(uintptr_t)read_type,
 810                                                         enabled_core_ids[i]);
 811                         ret = write_keys(write_type);
 812                         writer_done = 1;
 813
 814                         if (ret < 0)
 815                                 goto err;
 816                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 817                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 818                                         goto err;
 819
 820                         unsigned long long cycles_per_lookup =
 821                                 rte_atomic64_read(&gread_cycles) /
 822                                 rte_atomic64_read(&greads);
 823                         rwc_perf_results->w_no_ks_r_miss[m][n]
 824                                                 = cycles_per_lookup;
 825                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 826                 }
 827         }
 828
 829 finish:
 830         rte_hash_free(tbl_rwc_test_param.h);
 831         return 0;
 832
 833 err:
 834         rte_eal_mp_wait_lcore();
 835         rte_hash_free(tbl_rwc_test_param.h);
 836         return -1;
 837 }
 838
 839 /*
 840  * Test lookup perf:
 841  * Reader(s) lookup keys present in the table and not likely to be on the
 842  * shift path  while 'Main' thread adds keys causing key-shifts.
 843  */
 844 static int
 845 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 846                                     int rwc_lf, int htm, int ext_bkt)
 847 {
 848         unsigned int n, m;
 849         uint64_t i;
 850         int use_jhash = 0;
 851         int ret;
 852         uint8_t write_type;
 853         uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
 854
 855         rte_atomic64_init(&greads);
 856         rte_atomic64_init(&gread_cycles);
 857
 858         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 859                 goto err;
 860         printf("\nTest: Hash add - key shift, Hash lookup - hit"
 861                " (non-shift-path)\n");
 862         for (m = 0; m < 2; m++) {
 863                 if (m == 1) {
 864                         printf("\n** With bulk-lookup **\n");
 865                         read_type |= BULK_LOOKUP;
 866                 }
 867                 for (n = 0; n < NUM_TEST; n++) {
 868                         unsigned int tot_lcore = rte_lcore_count();
 869                         if (tot_lcore < rwc_core_cnt[n] + 1)
 870                                 goto finish;
 871
 872                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 873
 874                         rte_atomic64_clear(&greads);
 875                         rte_atomic64_clear(&gread_cycles);
 876
 877                         rte_hash_reset(tbl_rwc_test_param.h);
 878                         writer_done = 0;
 879                         write_type = WRITE_NO_KEY_SHIFT;
 880                         if (write_keys(write_type) < 0)
 881                                 goto err;
 882                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 883                                 rte_eal_remote_launch(test_rwc_reader,
 884                                                 (void *)(uintptr_t)read_type,
 885                                                         enabled_core_ids[i]);
 886                         write_type = WRITE_KEY_SHIFT;
 887                         ret = write_keys(write_type);
 888                         writer_done = 1;
 889
 890                         if (ret < 0)
 891                                 goto err;
 892                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 893                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 894                                         goto err;
 895
 896                         unsigned long long cycles_per_lookup =
 897                                 rte_atomic64_read(&gread_cycles) /
 898                                 rte_atomic64_read(&greads);
 899                         rwc_perf_results->w_ks_r_hit_nsp[m][n]
 900                                                 = cycles_per_lookup;
 901                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 902                 }
 903         }
 904
 905 finish:
 906         rte_hash_free(tbl_rwc_test_param.h);
 907         return 0;
 908
 909 err:
 910         rte_eal_mp_wait_lcore();
 911         rte_hash_free(tbl_rwc_test_param.h);
 912         return -1;
 913 }
 914
 915 /*
 916  * Test lookup perf:
 917  * Reader(s) lookup keys present in the table and likely on the shift-path while
 918  * 'Main' thread adds keys causing key-shifts.
 919  */
 920 static int
 921 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 922                                 int htm, int ext_bkt)
 923 {
 924         unsigned int n, m;
 925         uint64_t i;
 926         int use_jhash = 0;
 927         int ret;
 928         uint8_t write_type;
 929         uint8_t read_type = READ_PASS_SHIFT_PATH;
 930
 931         rte_atomic64_init(&greads);
 932         rte_atomic64_init(&gread_cycles);
 933
 934         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 935                 goto err;
 936         printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
 937                "\n");
 938
 939         for (m = 0; m < 2; m++) {
 940                 if (m == 1) {
 941                         printf("\n** With bulk-lookup **\n");
 942                         read_type |= BULK_LOOKUP;
 943                 }
 944                 for (n = 0; n < NUM_TEST; n++) {
 945                         unsigned int tot_lcore = rte_lcore_count();
 946                         if (tot_lcore < rwc_core_cnt[n] + 1)
 947                                 goto finish;
 948
 949                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 950                         rte_atomic64_clear(&greads);
 951                         rte_atomic64_clear(&gread_cycles);
 952
 953                         rte_hash_reset(tbl_rwc_test_param.h);
 954                         writer_done = 0;
 955                         write_type = WRITE_NO_KEY_SHIFT;
 956                         if (write_keys(write_type) < 0)
 957                                 goto err;
 958                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 959                                 rte_eal_remote_launch(test_rwc_reader,
 960                                                 (void *)(uintptr_t)read_type,
 961                                                 enabled_core_ids[i]);
 962                         write_type = WRITE_KEY_SHIFT;
 963                         ret = write_keys(write_type);
 964                         writer_done = 1;
 965
 966                         if (ret < 0)
 967                                 goto err;
 968                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 969                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 970                                         goto err;
 971
 972                         unsigned long long cycles_per_lookup =
 973                                 rte_atomic64_read(&gread_cycles) /
 974                                 rte_atomic64_read(&greads);
 975                         rwc_perf_results->w_ks_r_hit_sp[m][n]
 976                                                 = cycles_per_lookup;
 977                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 978                 }
 979         }
 980
 981 finish:
 982         rte_hash_free(tbl_rwc_test_param.h);
 983         return 0;
 984
 985 err:
 986         rte_eal_mp_wait_lcore();
 987         rte_hash_free(tbl_rwc_test_param.h);
 988         return -1;
 989 }
 990
 991 /*
 992  * Test lookup perf:
 993  * Reader(s) lookup keys absent in the table while
 994  * 'Main' thread adds keys causing key-shifts.
 995  */
 996 static int
 997 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 998                              htm, int ext_bkt)
 999 {
1000         unsigned int n, m;
1001         uint64_t i;
1002         int use_jhash = 0;
1003         int ret;
1004         uint8_t write_type;
1005         uint8_t read_type = READ_FAIL;
1006
1007         rte_atomic64_init(&greads);
1008         rte_atomic64_init(&gread_cycles);
1009
1010         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1011                 goto err;
1012         printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1013         for (m = 0; m < 2; m++) {
1014                 if (m == 1) {
1015                         printf("\n** With bulk-lookup **\n");
1016                         read_type |= BULK_LOOKUP;
1017                 }
1018                 for (n = 0; n < NUM_TEST; n++) {
1019                         unsigned int tot_lcore = rte_lcore_count();
1020                         if (tot_lcore < rwc_core_cnt[n] + 1)
1021                                 goto finish;
1022
1023                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1024
1025                         rte_atomic64_clear(&greads);
1026                         rte_atomic64_clear(&gread_cycles);
1027
1028                         rte_hash_reset(tbl_rwc_test_param.h);
1029                         writer_done = 0;
1030                         write_type = WRITE_NO_KEY_SHIFT;
1031                         if (write_keys(write_type) < 0)
1032                                 goto err;
1033                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1034                                 rte_eal_remote_launch(test_rwc_reader,
1035                                                 (void *)(uintptr_t)read_type,
1036                                                         enabled_core_ids[i]);
1037                         write_type = WRITE_KEY_SHIFT;
1038                         ret = write_keys(write_type);
1039                         writer_done = 1;
1040
1041                         if (ret < 0)
1042                                 goto err;
1043                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1044                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1045                                         goto err;
1046
1047                         unsigned long long cycles_per_lookup =
1048                                 rte_atomic64_read(&gread_cycles) /
1049                                 rte_atomic64_read(&greads);
1050                         rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1051                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1052                 }
1053         }
1054
1055 finish:
1056         rte_hash_free(tbl_rwc_test_param.h);
1057         return 0;
1058
1059 err:
1060         rte_eal_mp_wait_lcore();
1061         rte_hash_free(tbl_rwc_test_param.h);
1062         return -1;
1063 }
1064
1065 /*
1066  * Test lookup perf for multi-writer:
1067  * Reader(s) lookup keys present in the table and likely on the shift-path while
1068  * Writers add keys causing key-shiftsi.
1069  * Writers are running in parallel, on different data plane cores.
1070  */
1071 static int
1072 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1073                            int htm, int ext_bkt)
1074 {
1075         unsigned int n, m, k;
1076         uint64_t i;
1077         int use_jhash = 0;
1078         uint8_t write_type;
1079         uint8_t read_type = READ_PASS_SHIFT_PATH;
1080
1081         rte_atomic64_init(&greads);
1082         rte_atomic64_init(&gread_cycles);
1083
1084         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1085                 goto err;
1086         printf("\nTest: Multi-add-lookup\n");
1087         uint8_t pos_core;
1088         for (m = 1; m < NUM_TEST; m++) {
1089                 /* Calculate keys added by each writer */
1090                 tbl_rwc_test_param.single_insert =
1091                         tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1092                 for (k = 0; k < 2; k++) {
1093                         if (k == 1) {
1094                                 printf("\n** With bulk-lookup **\n");
1095                                 read_type |= BULK_LOOKUP;
1096                         }
1097                         for (n = 0; n < NUM_TEST; n++) {
1098                                 unsigned int tot_lcore  = rte_lcore_count();
1099                                 if (tot_lcore < (rwc_core_cnt[n] +
1100                                      rwc_core_cnt[m] + 1))
1101                                         goto finish;
1102
1103                                 printf("\nNumber of writers: %u",
1104                                        rwc_core_cnt[m]);
1105                                 printf("\nNumber of readers: %u\n",
1106                                        rwc_core_cnt[n]);
1107
1108                                 rte_atomic64_clear(&greads);
1109                                 rte_atomic64_clear(&gread_cycles);
1110
1111                                 rte_hash_reset(tbl_rwc_test_param.h);
1112                                 writer_done = 0;
1113                                 write_type = WRITE_NO_KEY_SHIFT;
1114                                 if (write_keys(write_type) < 0)
1115                                         goto err;
1116
1117                                 /* Launch reader(s) */
1118                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1119                                         rte_eal_remote_launch(test_rwc_reader,
1120                                                 (void *)(uintptr_t)read_type,
1121                                                 enabled_core_ids[i]);
1122                                 write_type = WRITE_KEY_SHIFT;
1123                                 pos_core = 0;
1124
1125                                 /* Launch writers */
1126                                 for (; i <= rwc_core_cnt[m]
1127                                      + rwc_core_cnt[n]; i++) {
1128                                         rte_eal_remote_launch
1129                                                 (test_rwc_multi_writer,
1130                                                 (void *)(uintptr_t)pos_core,
1131                                                 enabled_core_ids[i]);
1132                                         pos_core++;
1133                                 }
1134
1135                                 /* Wait for writers to complete */
1136                                 for (i = rwc_core_cnt[n] + 1;
1137                                      i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1138                                      i++)
1139                                         rte_eal_wait_lcore(enabled_core_ids[i]);
1140
1141                                 writer_done = 1;
1142
1143                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1144                                         if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1145                                                 goto err;
1146
1147                                 unsigned long long cycles_per_lookup =
1148                                         rte_atomic64_read(&gread_cycles)
1149                                         / rte_atomic64_read(&greads);
1150                                 rwc_perf_results->multi_rw[m][k][n]
1151                                         = cycles_per_lookup;
1152                                 printf("Cycles per lookup: %llu\n",
1153                                        cycles_per_lookup);
1154                         }
1155                 }
1156         }
1157
1158 finish:
1159         rte_hash_free(tbl_rwc_test_param.h);
1160         return 0;
1161
1162 err:
1163         rte_eal_mp_wait_lcore();
1164         rte_hash_free(tbl_rwc_test_param.h);
1165         return -1;
1166 }
1167
1168 /*
1169  * Test lookup perf:
1170  * Reader(s) lookup keys present in the extendable bkt.
1171  */
1172 static int
1173 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1174                                 int rwc_lf, int htm, int ext_bkt)
1175 {
1176         unsigned int n, m;
1177         uint64_t i;
1178         int use_jhash = 0;
1179         uint8_t write_type;
1180         uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1181
1182         rte_atomic64_init(&greads);
1183         rte_atomic64_init(&gread_cycles);
1184
1185         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1186                 goto err;
1187         printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1188         for (m = 0; m < 2; m++) {
1189                 if (m == 1) {
1190                         printf("\n** With bulk-lookup **\n");
1191                         read_type |= BULK_LOOKUP;
1192                 }
1193                 for (n = 0; n < NUM_TEST; n++) {
1194                         unsigned int tot_lcore = rte_lcore_count();
1195                         if (tot_lcore < rwc_core_cnt[n] + 1)
1196                                 goto finish;
1197
1198                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1199
1200                         rte_atomic64_clear(&greads);
1201                         rte_atomic64_clear(&gread_cycles);
1202
1203                         rte_hash_reset(tbl_rwc_test_param.h);
1204                         write_type = WRITE_NO_KEY_SHIFT;
1205                         if (write_keys(write_type) < 0)
1206                                 goto err;
1207                         write_type = WRITE_KEY_SHIFT;
1208                         if (write_keys(write_type) < 0)
1209                                 goto err;
1210                         writer_done = 0;
1211                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1212                                 rte_eal_remote_launch(test_rwc_reader,
1213                                                 (void *)(uintptr_t)read_type,
1214                                                         enabled_core_ids[i]);
1215                         for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1216                              i++) {
1217                                 if (rte_hash_del_key(tbl_rwc_test_param.h,
1218                                         tbl_rwc_test_param.keys_ks_extbkt + i)
1219                                                         < 0) {
1220                                         printf("Delete Failed: %u\n",
1221                                         tbl_rwc_test_param.keys_ks_extbkt[i]);
1222                                         goto err;
1223                                 }
1224                         }
1225                         writer_done = 1;
1226
1227                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1228                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1229                                         goto err;
1230
1231                         unsigned long long cycles_per_lookup =
1232                                 rte_atomic64_read(&gread_cycles) /
1233                                 rte_atomic64_read(&greads);
1234                         rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1235                                                 = cycles_per_lookup;
1236                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1237                 }
1238         }
1239
1240 finish:
1241         rte_hash_free(tbl_rwc_test_param.h);
1242         return 0;
1243
1244 err:
1245         rte_eal_mp_wait_lcore();
1246         rte_hash_free(tbl_rwc_test_param.h);
1247         return -1;
1248 }
1249
1250 static int
1251 test_hash_readwrite_lf_main(void)
1252 {
1253         /*
1254          * Variables used to choose different tests.
1255          * rwc_lf indicates if read-write concurrency lock-free support is
1256          * enabled.
1257          * htm indicates if Hardware transactional memory support is enabled.
1258          */
1259         int rwc_lf = 0;
1260         int htm;
1261         int ext_bkt = 0;
1262
1263         if (rte_lcore_count() < 2) {
1264                 printf("Not enough cores for hash_readwrite_lf_autotest, expecting at least 2\n");
1265                 return TEST_SKIPPED;
1266         }
1267
1268         setlocale(LC_NUMERIC, "");
1269
1270         /* Reset tbl_rwc_test_param to discard values from previous run */
1271         memset(&tbl_rwc_test_param, 0, sizeof(tbl_rwc_test_param));
1272
1273         if (rte_tm_supported())
1274                 htm = 1;
1275         else
1276                 htm = 0;
1277
1278         if (generate_keys() != 0)
1279                 return -1;
1280         if (get_enabled_cores_list() != 0)
1281                 return -1;
1282
1283         if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1284                 rwc_lf = 1;
1285                 ext_bkt = 1;
1286                 printf("Test lookup with read-write concurrency lock free support"
1287                        " enabled\n");
1288                 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1289                                                         htm, ext_bkt) < 0)
1290                         return -1;
1291                 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1292                                                         htm, ext_bkt) < 0)
1293                         return -1;
1294                 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1295                                                         htm, ext_bkt) < 0)
1296                         return -1;
1297                 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1298                                                         htm, ext_bkt) < 0)
1299                         return -1;
1300                 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1301                                                  ext_bkt) < 0)
1302                         return -1;
1303                 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1304                                                ext_bkt) < 0)
1305                         return -1;
1306                 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1307                                                         htm, ext_bkt) < 0)
1308                         return -1;
1309         }
1310         printf("\nTest lookup with read-write concurrency lock free support"
1311                " disabled\n");
1312         rwc_lf = 0;
1313         if (!htm) {
1314                 printf("With HTM Disabled\n");
1315                 if (!RUN_WITH_HTM_DISABLED) {
1316                         printf("Enable RUN_WITH_HTM_DISABLED to test with"
1317                                " lock-free disabled");
1318                         goto results;
1319                 }
1320         } else
1321                 printf("With HTM Enabled\n");
1322         if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1323                                            ext_bkt) < 0)
1324                 return -1;
1325         if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1326                                                 ext_bkt) < 0)
1327                 return -1;
1328         if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1329                                                 htm, ext_bkt) < 0)
1330                 return -1;
1331         if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1332                                                 ext_bkt) < 0)
1333                 return -1;
1334         if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1335                                          ext_bkt) < 0)
1336                 return -1;
1337         if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1338                                                         ext_bkt) < 0)
1339                 return -1;
1340         if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1341                                                 htm, ext_bkt) < 0)
1342                 return -1;
1343 results:
1344         printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1345         int i, j, k;
1346         for (j = 0; j < 2; j++) {
1347                 if (j == 1)
1348                         printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1349                                "**********#######\n\n");
1350                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1351                         "\t\t\t\t_________________\n");
1352                 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1353                        "\t\t\tCycles per lookup\n");
1354                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1355                        "\t\t\t_________________\n");
1356                 for (i = 0; i < NUM_TEST; i++) {
1357                         printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1358                         printf("Enabled\t\t");
1359                         printf("N/A\t\t");
1360                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1361                                 "%u\n\t\t\t\t\t\t\t\t",
1362                                 rwc_lf_results.w_no_ks_r_hit[j][i]);
1363                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1364                                 "%u\n\t\t\t\t\t\t\t\t",
1365                                 rwc_lf_results.w_no_ks_r_miss[j][i]);
1366                         printf("Hash add - key-shifts, lookup - hit"
1367                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1368                                rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1369                         printf("Hash add - key-shifts, lookup - hit "
1370                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1371                                rwc_lf_results.w_ks_r_hit_sp[j][i]);
1372                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1373                                 "%u\n\t\t\t\t\t\t\t\t",
1374                                 rwc_lf_results.w_ks_r_miss[j][i]);
1375                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1376                                 "%u\n\n\t\t\t\t",
1377                                 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1378
1379                         printf("Disabled\t");
1380                         if (htm)
1381                                 printf("Enabled\t\t");
1382                         else
1383                                 printf("Disabled\t");
1384                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1385                                 "%u\n\t\t\t\t\t\t\t\t",
1386                                 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1387                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1388                                 "%u\n\t\t\t\t\t\t\t\t",
1389                                 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1390                         printf("Hash add - key-shifts, lookup - hit "
1391                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1392                                rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1393                         printf("Hash add - key-shifts, lookup - hit "
1394                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1395                                rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1396                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1397                                "%u\n\t\t\t\t\t\t\t\t",
1398                                rwc_non_lf_results.w_ks_r_miss[j][i]);
1399                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1400                                 "%u\n",
1401                                 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1402
1403                         printf("_______\t\t_______\t\t_________\t___\t\t"
1404                                "_________\t\t\t\t\t\t_________________\n");
1405                 }
1406
1407                 for (i = 1; i < NUM_TEST; i++) {
1408                         for (k = 0; k < NUM_TEST; k++) {
1409                                 printf("%u", rwc_core_cnt[i]);
1410                                 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1411                                 printf("Enabled\t\t");
1412                                 printf("N/A\t\t");
1413                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1414                                        "\t\t",
1415                                        rwc_lf_results.multi_rw[i][j][k]);
1416                                 printf("Disabled\t");
1417                                 if (htm)
1418                                         printf("Enabled\t\t");
1419                                 else
1420                                         printf("Disabled\t");
1421                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1422                                        rwc_non_lf_results.multi_rw[i][j][k]);
1423
1424                                 printf("_______\t\t_______\t\t_________\t___"
1425                                        "\t\t_________\t\t\t\t\t\t"
1426                                        "_________________\n");
1427                         }
1428                 }
1429         }
1430         rte_free(tbl_rwc_test_param.keys);
1431         rte_free(tbl_rwc_test_param.keys_no_ks);
1432         rte_free(tbl_rwc_test_param.keys_ks);
1433         rte_free(tbl_rwc_test_param.keys_absent);
1434         rte_free(tbl_rwc_test_param.keys_shift_path);
1435         rte_free(tbl_rwc_test_param.keys_non_shift_path);
1436         rte_free(tbl_rwc_test_param.keys_ext_bkt);
1437         rte_free(tbl_rwc_test_param.keys_ks_extbkt);
1438         return 0;
1439 }
1440
1441 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);