app/test/test_hash_readwrite_lf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Arm Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
  20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
  21 #endif
  22
  23 #define BULK_LOOKUP_SIZE 32
  24
  25 #define RUN_WITH_HTM_DISABLED 0
  26
  27 #if (RUN_WITH_HTM_DISABLED)
  28
  29 #define TOTAL_ENTRY (5*1024)
  30 #define TOTAL_INSERT (5*1024)
  31
  32 #else
  33
  34 #define TOTAL_ENTRY (4*1024*1024)
  35 #define TOTAL_INSERT (4*1024*1024)
  36
  37 #endif
  38
  39 #define READ_FAIL 1
  40 #define READ_PASS_NO_KEY_SHIFTS 2
  41 #define READ_PASS_SHIFT_PATH 4
  42 #define READ_PASS_NON_SHIFT_PATH 8
  43 #define BULK_LOOKUP 16
  44 #define READ_PASS_KEY_SHIFTS_EXTBKT 32
  45
  46 #define WRITE_NO_KEY_SHIFT 0
  47 #define WRITE_KEY_SHIFT 1
  48 #define WRITE_EXT_BKT 2
  49
  50 #define NUM_TEST 3
  51 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
  52
  53 struct rwc_perf {
  54         uint32_t w_no_ks_r_hit[2][NUM_TEST];
  55         uint32_t w_no_ks_r_miss[2][NUM_TEST];
  56         uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
  57         uint32_t w_ks_r_hit_sp[2][NUM_TEST];
  58         uint32_t w_ks_r_miss[2][NUM_TEST];
  59         uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
  60         uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
  61 };
  62
  63 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
  64
  65 struct {
  66         uint32_t *keys;
  67         uint32_t *keys_no_ks;
  68         uint32_t *keys_ks;
  69         uint32_t *keys_absent;
  70         uint32_t *keys_shift_path;
  71         uint32_t *keys_non_shift_path;
  72         uint32_t *keys_ext_bkt;
  73         uint32_t *keys_ks_extbkt;
  74         uint32_t count_keys_no_ks;
  75         uint32_t count_keys_ks;
  76         uint32_t count_keys_absent;
  77         uint32_t count_keys_shift_path;
  78         uint32_t count_keys_non_shift_path;
  79         uint32_t count_keys_extbkt;
  80         uint32_t count_keys_ks_extbkt;
  81         uint32_t single_insert;
  82         struct rte_hash *h;
  83 } tbl_rwc_test_param;
  84
  85 static rte_atomic64_t gread_cycles;
  86 static rte_atomic64_t greads;
  87
  88 static volatile uint8_t writer_done;
  89
  90 uint16_t enabled_core_ids[RTE_MAX_LCORE];
  91
  92 uint8_t *scanned_bkts;
  93
  94 static inline uint16_t
  95 get_short_sig(const hash_sig_t hash)
  96 {
  97         return hash >> 16;
  98 }
  99
 100 static inline uint32_t
 101 get_prim_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 102                       const hash_sig_t hash)
 103 {
 104         uint32_t num_buckets;
 105         uint32_t bucket_bitmask;
 106         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 107         bucket_bitmask = num_buckets - 1;
 108         return hash & bucket_bitmask;
 109 }
 110
 111 static inline uint32_t
 112 get_alt_bucket_index(__attribute__((unused)) const struct rte_hash *h,
 113                         uint32_t cur_bkt_idx, uint16_t sig)
 114 {
 115         uint32_t num_buckets;
 116         uint32_t bucket_bitmask;
 117         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 118         bucket_bitmask = num_buckets - 1;
 119         return (cur_bkt_idx ^ sig) & bucket_bitmask;
 120 }
 121
 122
 123 static inline int
 124 get_enabled_cores_list(void)
 125 {
 126         uint32_t i = 0;
 127         uint16_t core_id;
 128         uint32_t max_cores = rte_lcore_count();
 129         RTE_LCORE_FOREACH(core_id) {
 130                 enabled_core_ids[i] = core_id;
 131                 i++;
 132         }
 133
 134         if (i != max_cores) {
 135                 printf("Number of enabled cores in list is different from "
 136                         "number given by rte_lcore_count()\n");
 137                 return -1;
 138         }
 139         return 0;
 140 }
 141
 142 static inline int
 143 check_bucket(uint32_t bkt_idx, uint32_t key)
 144 {
 145         uint32_t iter;
 146         uint32_t prev_iter;
 147         uint32_t diff;
 148         uint32_t count = 0;
 149         const void *next_key;
 150         void *next_data;
 151
 152         /* Temporary bucket to hold the keys */
 153         uint32_t keys_in_bkt[8];
 154
 155         iter = bkt_idx * 8;
 156         prev_iter = iter;
 157         while (rte_hash_iterate(tbl_rwc_test_param.h,
 158                         &next_key, &next_data, &iter) >= 0) {
 159
 160                 /* Check for duplicate entries */
 161                 if (*(const uint32_t *)next_key == key)
 162                         return 1;
 163
 164                 /* Identify if there is any free entry in the bucket */
 165                 diff = iter - prev_iter;
 166                 if (diff > 1)
 167                         break;
 168
 169                 prev_iter = iter;
 170                 keys_in_bkt[count] = *(const uint32_t *)next_key;
 171                 count++;
 172
 173                 /* All entries in the bucket are occupied */
 174                 if (count == 8) {
 175
 176                         /*
 177                          * Check if bucket was not scanned before, to avoid
 178                          * duplicate keys.
 179                          */
 180                         if (scanned_bkts[bkt_idx] == 0) {
 181                                 /*
 182                                  * Since this bucket (pointed to by bkt_idx) is
 183                                  * full, it is likely that key(s) in this
 184                                  * bucket will be on the shift path, when
 185                                  * collision occurs. Thus, add it to
 186                                  * keys_shift_path.
 187                                  */
 188                                 memcpy(tbl_rwc_test_param.keys_shift_path +
 189                                         tbl_rwc_test_param.count_keys_shift_path
 190                                         , keys_in_bkt, 32);
 191                                 tbl_rwc_test_param.count_keys_shift_path += 8;
 192                                 scanned_bkts[bkt_idx] = 1;
 193                         }
 194                         return -1;
 195                 }
 196         }
 197         return 0;
 198 }
 199
 200 static int
 201 generate_keys(void)
 202 {
 203         uint32_t *keys = NULL;
 204         uint32_t *keys_no_ks = NULL;
 205         uint32_t *keys_ks = NULL;
 206         uint32_t *keys_absent = NULL;
 207         uint32_t *keys_non_shift_path = NULL;
 208         uint32_t *keys_ext_bkt = NULL;
 209         uint32_t *keys_ks_extbkt = NULL;
 210         uint32_t *found = NULL;
 211         uint32_t count_keys_no_ks = 0;
 212         uint32_t count_keys_ks = 0;
 213         uint32_t count_keys_extbkt = 0;
 214         uint32_t i;
 215
 216         /*
 217          * keys will consist of a) keys whose addition to the hash table
 218          * will result in shifting of the existing keys to their alternate
 219          * locations b) keys whose addition to the hash table will not result
 220          * in shifting of the existing keys.
 221          */
 222         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 223         if (keys == NULL) {
 224                 printf("RTE_MALLOC failed\n");
 225                 goto err;
 226         }
 227
 228         /*
 229          * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
 230          * will NOT result in shifting of the existing keys to their alternate
 231          * locations. Roughly around 900K keys.
 232          */
 233         keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 234         if (keys_no_ks == NULL) {
 235                 printf("RTE_MALLOC failed\n");
 236                 goto err;
 237         }
 238
 239         /*
 240          * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
 241          * result in shifting of the existing keys to their alternate locations.
 242          * Roughly around 146K keys. There might be repeating keys. More code is
 243          * required to filter out these keys which will complicate the test case
 244          */
 245         keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 246         if (keys_ks == NULL) {
 247                 printf("RTE_MALLOC failed\n");
 248                 goto err;
 249         }
 250
 251         /* Used to identify keys not inserted in the hash table */
 252         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 253         if (found == NULL) {
 254                 printf("RTE_MALLOC failed\n");
 255                 goto err;
 256         }
 257
 258         /*
 259          * This consist of keys not inserted to the hash table.
 260          * Used to test perf of lookup on keys that do not exist in the table.
 261          */
 262         keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 263         if (keys_absent == NULL) {
 264                 printf("RTE_MALLOC failed\n");
 265                 goto err;
 266         }
 267
 268         /*
 269          * This consist of keys which are likely to be on the shift
 270          * path (i.e. being moved to alternate location), when collision occurs
 271          * on addition of a key to an already full primary bucket.
 272          * Used to test perf of lookup on keys that are on the shift path.
 273          */
 274         tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
 275                                                         TOTAL_INSERT, 0);
 276         if (tbl_rwc_test_param.keys_shift_path == NULL) {
 277                 printf("RTE_MALLOC failed\n");
 278                 goto err;
 279         }
 280
 281         /*
 282          * This consist of keys which are never on the shift
 283          * path (i.e. being moved to alternate location), when collision occurs
 284          * on addition of a key to an already full primary bucket.
 285          * Used to test perf of lookup on keys that are not on the shift path.
 286          */
 287         keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
 288                                          0);
 289         if (keys_non_shift_path == NULL) {
 290                 printf("RTE_MALLOC failed\n");
 291                 goto err;
 292         }
 293
 294         /*
 295          * This consist of keys which will be stored in extended buckets
 296          */
 297         keys_ext_bkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 298         if (keys_ext_bkt == NULL) {
 299                 printf("RTE_MALLOC failed\n");
 300                 goto err;
 301         }
 302
 303         /*
 304          * This consist of keys which when deleted causes shifting of keys
 305          * in extended buckets to respective secondary buckets
 306          */
 307         keys_ks_extbkt = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 308         if (keys_ks_extbkt == NULL) {
 309                 printf("RTE_MALLOC failed\n");
 310                 goto err;
 311         }
 312
 313         hash_sig_t sig;
 314         uint32_t prim_bucket_idx;
 315         uint32_t sec_bucket_idx;
 316         uint16_t short_sig;
 317         uint32_t num_buckets;
 318         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 319         int ret;
 320
 321         /*
 322          * Used to mark bkts in which at least one key was shifted to its
 323          * alternate location
 324          */
 325         scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
 326         if (scanned_bkts == NULL) {
 327                 printf("RTE_MALLOC failed\n");
 328                 goto err;
 329         }
 330
 331         tbl_rwc_test_param.keys = keys;
 332         tbl_rwc_test_param.keys_no_ks = keys_no_ks;
 333         tbl_rwc_test_param.keys_ks = keys_ks;
 334         tbl_rwc_test_param.keys_absent = keys_absent;
 335         tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
 336         tbl_rwc_test_param.keys_ext_bkt = keys_ext_bkt;
 337         tbl_rwc_test_param.keys_ks_extbkt = keys_ks_extbkt;
 338         /* Generate keys by adding previous two keys, neglect overflow */
 339         printf("Generating keys...\n");
 340         keys[0] = 0;
 341         keys[1] = 1;
 342         for (i = 2; i < TOTAL_INSERT; i++)
 343                 keys[i] = keys[i-1] + keys[i-2];
 344
 345         /* Segregate keys into keys_no_ks and keys_ks */
 346         for (i = 0; i < TOTAL_INSERT; i++) {
 347                 /* Check if primary bucket has space.*/
 348                 sig = rte_hash_hash(tbl_rwc_test_param.h,
 349                                         tbl_rwc_test_param.keys+i);
 350                 prim_bucket_idx = get_prim_bucket_index(tbl_rwc_test_param.h,
 351                                                         sig);
 352                 ret = check_bucket(prim_bucket_idx, keys[i]);
 353                 if (ret < 0) {
 354                         /*
 355                          * Primary bucket is full, this key will result in
 356                          * shifting of the keys to their alternate locations.
 357                          */
 358                         keys_ks[count_keys_ks] = keys[i];
 359                         count_keys_ks++;
 360                 } else if (ret == 0) {
 361                         /*
 362                          * Primary bucket has space, this key will not result in
 363                          * shifting of the keys. Hence, add key to the table.
 364                          */
 365                         ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
 366                                                         keys+i,
 367                                                         (void *)((uintptr_t)i));
 368                         if (ret < 0) {
 369                                 printf("writer failed %"PRIu32"\n", i);
 370                                 break;
 371                         }
 372                         keys_no_ks[count_keys_no_ks] = keys[i];
 373                         count_keys_no_ks++;
 374                 }
 375         }
 376
 377         for (i = 0; i < count_keys_no_ks; i++) {
 378                 /*
 379                  * Identify keys in keys_no_ks with value less than
 380                  * 4M (HTM enabled) OR 5K (HTM disabled)
 381                  */
 382                 if (keys_no_ks[i] < TOTAL_INSERT)
 383                         found[keys_no_ks[i]]++;
 384         }
 385
 386         for (i = 0; i < count_keys_ks; i++) {
 387                 /*
 388                  * Identify keys in keys_ks with value less than
 389                  * 4M (HTM enabled) OR 5K (HTM disabled)
 390                  */
 391                 if (keys_ks[i] < TOTAL_INSERT)
 392                         found[keys_ks[i]]++;
 393         }
 394
 395         uint32_t count_keys_absent = 0;
 396         for (i = 0; i < TOTAL_INSERT; i++) {
 397                 /*
 398                  * Identify missing keys between 0 and
 399                  * 4M (HTM enabled) OR 5K (HTM disabled)
 400                  */
 401                 if (found[i] == 0)
 402                         keys_absent[count_keys_absent++] = i;
 403         }
 404
 405         /* Find keys that will not be on the shift path */
 406         uint32_t iter;
 407         const void *next_key;
 408         void *next_data;
 409         uint32_t count = 0;
 410         for (i = 0; i < num_buckets; i++) {
 411                 /* Check bucket for no keys shifted to alternate locations */
 412                 if (scanned_bkts[i] == 0) {
 413                         iter = i * 8;
 414                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 415                                 &next_key, &next_data, &iter) >= 0) {
 416
 417                                 /* Check if key belongs to the current bucket */
 418                                 if (i >= (iter-1)/8)
 419                                         keys_non_shift_path[count++]
 420                                                 = *(const uint32_t *)next_key;
 421                                 else
 422                                         break;
 423                         }
 424                 }
 425         }
 426
 427         tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
 428         tbl_rwc_test_param.count_keys_ks = count_keys_ks;
 429         tbl_rwc_test_param.count_keys_absent = count_keys_absent;
 430         tbl_rwc_test_param.count_keys_non_shift_path = count;
 431
 432         memset(scanned_bkts, 0, num_buckets);
 433         count = 0;
 434         /* Find keys that will be in extended buckets */
 435         for (i = 0; i < count_keys_ks; i++) {
 436                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys_ks + i);
 437                 if (ret < 0) {
 438                         /* Key will be added to ext bkt */
 439                         keys_ext_bkt[count_keys_extbkt++] = keys_ks[i];
 440                         /* Sec bkt to be added to keys_ks_extbkt */
 441                         sig = rte_hash_hash(tbl_rwc_test_param.h,
 442                                         tbl_rwc_test_param.keys_ks + i);
 443                         prim_bucket_idx = get_prim_bucket_index(
 444                                                 tbl_rwc_test_param.h, sig);
 445                         short_sig = get_short_sig(sig);
 446                         sec_bucket_idx = get_alt_bucket_index(
 447                                                 tbl_rwc_test_param.h,
 448                                                 prim_bucket_idx, short_sig);
 449                         if (scanned_bkts[sec_bucket_idx] == 0)
 450                                 scanned_bkts[sec_bucket_idx] = 1;
 451                 }
 452         }
 453
 454         /* Find keys that will shift keys in ext bucket*/
 455         for (i = 0; i < num_buckets; i++) {
 456                 if (scanned_bkts[i] == 1) {
 457                         iter = i * 8;
 458                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 459                                 &next_key, &next_data, &iter) >= 0) {
 460                                 /* Check if key belongs to the current bucket */
 461                                 if (i >= (iter-1)/8)
 462                                         keys_ks_extbkt[count++]
 463                                                 = *(const uint32_t *)next_key;
 464                                 else
 465                                         break;
 466                         }
 467                 }
 468         }
 469
 470         tbl_rwc_test_param.count_keys_ks_extbkt = count;
 471         tbl_rwc_test_param.count_keys_extbkt = count_keys_extbkt;
 472
 473         printf("\nCount of keys NOT causing shifting of existing keys to "
 474         "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
 475         printf("\nCount of keys causing shifting of existing keys to alternate "
 476                 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
 477         printf("Count of absent keys that will never be added to the hash "
 478                 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
 479         printf("Count of keys likely to be on the shift path: %d\n\n",
 480                tbl_rwc_test_param.count_keys_shift_path);
 481         printf("Count of keys not likely to be on the shift path: %d\n\n",
 482                tbl_rwc_test_param.count_keys_non_shift_path);
 483         printf("Count of keys in extended buckets: %d\n\n",
 484                tbl_rwc_test_param.count_keys_extbkt);
 485         printf("Count of keys shifting keys in ext buckets: %d\n\n",
 486                tbl_rwc_test_param.count_keys_ks_extbkt);
 487
 488         rte_free(found);
 489         rte_hash_free(tbl_rwc_test_param.h);
 490         return 0;
 491
 492 err:
 493         rte_free(keys);
 494         rte_free(keys_no_ks);
 495         rte_free(keys_ks);
 496         rte_free(keys_absent);
 497         rte_free(found);
 498         rte_free(tbl_rwc_test_param.keys_shift_path);
 499         rte_free(keys_ext_bkt);
 500         rte_free(keys_ks_extbkt);
 501         rte_free(scanned_bkts);
 502         return -1;
 503 }
 504
 505 static int
 506 init_params(int rwc_lf, int use_jhash, int htm, int ext_bkt)
 507 {
 508         struct rte_hash *handle;
 509
 510         struct rte_hash_parameters hash_params = {
 511                 .entries = TOTAL_ENTRY,
 512                 .key_len = sizeof(uint32_t),
 513                 .hash_func_init_val = 0,
 514                 .socket_id = rte_socket_id(),
 515         };
 516
 517         if (use_jhash)
 518                 hash_params.hash_func = rte_jhash;
 519         else
 520                 hash_params.hash_func = rte_hash_crc;
 521
 522         if (rwc_lf)
 523                 hash_params.extra_flag =
 524                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
 525                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 526         else if (htm)
 527                 hash_params.extra_flag =
 528                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 529                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 530                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 531         else
 532                 hash_params.extra_flag =
 533                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 534                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 535
 536         if (ext_bkt)
 537                 hash_params.extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
 538
 539         hash_params.name = "tests";
 540
 541         handle = rte_hash_create(&hash_params);
 542         if (handle == NULL) {
 543                 printf("hash creation failed");
 544                 return -1;
 545         }
 546
 547         tbl_rwc_test_param.h = handle;
 548         return 0;
 549 }
 550
 551 static int
 552 test_rwc_reader(__attribute__((unused)) void *arg)
 553 {
 554         uint32_t i, j;
 555         int ret;
 556         uint64_t begin, cycles;
 557         uint32_t loop_cnt = 0;
 558         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 559         uint32_t read_cnt;
 560         uint32_t *keys;
 561         uint32_t extra_keys;
 562         int32_t *pos;
 563         void *temp_a[BULK_LOOKUP_SIZE];
 564
 565         /* Used to identify keys not inserted in the hash table */
 566         pos = rte_malloc(NULL, sizeof(uint32_t) * BULK_LOOKUP_SIZE, 0);
 567         if (pos == NULL) {
 568                 printf("RTE_MALLOC failed\n");
 569                 return -1;
 570         }
 571
 572         if (read_type & READ_FAIL) {
 573                 keys = tbl_rwc_test_param.keys_absent;
 574                 read_cnt = tbl_rwc_test_param.count_keys_absent;
 575         } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
 576                 keys = tbl_rwc_test_param.keys_no_ks;
 577                 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
 578         } else if (read_type & READ_PASS_SHIFT_PATH) {
 579                 keys = tbl_rwc_test_param.keys_shift_path;
 580                 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
 581         } else if (read_type & READ_PASS_KEY_SHIFTS_EXTBKT) {
 582                 keys = tbl_rwc_test_param.keys_ext_bkt;
 583                 read_cnt = tbl_rwc_test_param.count_keys_extbkt;
 584         } else {
 585                 keys = tbl_rwc_test_param.keys_non_shift_path;
 586                 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
 587         }
 588
 589         extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
 590
 591         begin = rte_rdtsc_precise();
 592         do {
 593                 if (read_type & BULK_LOOKUP) {
 594                         for (i = 0; i < (read_cnt - extra_keys);
 595                              i += BULK_LOOKUP_SIZE) {
 596                                 /* Array of  pointer to the list of keys */
 597                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 598                                         temp_a[j] = keys + i + j;
 599                                 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 600                                                    (const void **)
 601                                                    ((uintptr_t)temp_a),
 602                                                    BULK_LOOKUP_SIZE, pos);
 603                                 /* Validate lookup result */
 604                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 605                                         if ((read_type & READ_FAIL &&
 606                                              pos[j] != -ENOENT) ||
 607                                             (!(read_type & READ_FAIL) &&
 608                                              pos[j] == -ENOENT)) {
 609                                                 printf("lookup failed!"
 610                                                        "%"PRIu32"\n",
 611                                                        keys[i + j]);
 612                                                 return -1;
 613                                         }
 614                         }
 615                         for (j = 0; j < extra_keys; j++)
 616                                 temp_a[j] = keys + i + j;
 617
 618                         rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 619                                            (const void **)
 620                                            ((uintptr_t)temp_a),
 621                                            extra_keys, pos);
 622                         for (j = 0; j < extra_keys; j++)
 623                                 if ((read_type & READ_FAIL &&
 624                                      pos[j] != -ENOENT) ||
 625                                     (!(read_type & READ_FAIL) &&
 626                                      pos[j] == -ENOENT)) {
 627                                         printf("lookup failed! %"PRIu32"\n",
 628                                                keys[i + j]);
 629                                         return -1;
 630                                 }
 631                 } else {
 632                         for (i = 0; i < read_cnt; i++) {
 633                                 ret = rte_hash_lookup
 634                                         (tbl_rwc_test_param.h, keys + i);
 635                                 if (((read_type & READ_FAIL) &&
 636                                      (ret != -ENOENT)) ||
 637                                     (!(read_type & READ_FAIL) &&
 638                                         ret == -ENOENT)) {
 639                                         printf("lookup failed! %"PRIu32"\n",
 640                                                keys[i]);
 641                                         return -1;
 642                                 }
 643                         }
 644                 }
 645                 loop_cnt++;
 646         } while (!writer_done);
 647
 648         cycles = rte_rdtsc_precise() - begin;
 649         rte_atomic64_add(&gread_cycles, cycles);
 650         rte_atomic64_add(&greads, read_cnt*loop_cnt);
 651         return 0;
 652 }
 653
 654 static int
 655 write_keys(uint8_t write_type)
 656 {
 657         uint32_t i;
 658         int ret;
 659         uint32_t key_cnt = 0;
 660         uint32_t *keys;
 661         if (write_type == WRITE_KEY_SHIFT) {
 662                 key_cnt = tbl_rwc_test_param.count_keys_ks;
 663                 keys = tbl_rwc_test_param.keys_ks;
 664         } else if (write_type == WRITE_NO_KEY_SHIFT) {
 665                 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
 666                 keys = tbl_rwc_test_param.keys_no_ks;
 667         } else if (write_type == WRITE_EXT_BKT) {
 668                 key_cnt = tbl_rwc_test_param.count_keys_extbkt;
 669                 keys = tbl_rwc_test_param.keys_ext_bkt;
 670         }
 671         for (i = 0; i < key_cnt; i++) {
 672                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
 673                 if ((write_type == WRITE_NO_KEY_SHIFT) && ret < 0) {
 674                         printf("writer failed %"PRIu32"\n", i);
 675                         return -1;
 676                 }
 677         }
 678         return 0;
 679 }
 680
 681 static int
 682 test_rwc_multi_writer(__attribute__((unused)) void *arg)
 683 {
 684         uint32_t i, offset;
 685         uint32_t pos_core = (uint32_t)((uintptr_t)arg);
 686         offset = pos_core * tbl_rwc_test_param.single_insert;
 687         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
 688                 rte_hash_add_key(tbl_rwc_test_param.h,
 689                                  tbl_rwc_test_param.keys_ks + i);
 690         return 0;
 691 }
 692
 693 /*
 694  * Test lookup perf:
 695  * Reader(s) lookup keys present in the table.
 696  */
 697 static int
 698 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 699                                 int htm, int ext_bkt)
 700 {
 701         unsigned int n, m;
 702         uint64_t i;
 703         int use_jhash = 0;
 704         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 705         uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
 706
 707         rte_atomic64_init(&greads);
 708         rte_atomic64_init(&gread_cycles);
 709
 710         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 711                 goto err;
 712         printf("\nTest: Hash add - no key-shifts, read - hit\n");
 713         for (m = 0; m < 2; m++) {
 714                 if (m == 1) {
 715                         printf("\n** With bulk-lookup **\n");
 716                         read_type |= BULK_LOOKUP;
 717                 }
 718                 for (n = 0; n < NUM_TEST; n++) {
 719                         unsigned int tot_lcore = rte_lcore_count();
 720                         if (tot_lcore < rwc_core_cnt[n] + 1)
 721                                 goto finish;
 722
 723                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 724
 725                         rte_atomic64_clear(&greads);
 726                         rte_atomic64_clear(&gread_cycles);
 727
 728                         rte_hash_reset(tbl_rwc_test_param.h);
 729                         writer_done = 0;
 730                         if (write_keys(write_type) < 0)
 731                                 goto err;
 732                         writer_done = 1;
 733                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 734                                 rte_eal_remote_launch(test_rwc_reader,
 735                                                 (void *)(uintptr_t)read_type,
 736                                                         enabled_core_ids[i]);
 737
 738                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 739                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 740                                         goto err;
 741
 742                         unsigned long long cycles_per_lookup =
 743                                 rte_atomic64_read(&gread_cycles) /
 744                                 rte_atomic64_read(&greads);
 745                         rwc_perf_results->w_no_ks_r_hit[m][n]
 746                                                 = cycles_per_lookup;
 747                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 748                 }
 749         }
 750
 751 finish:
 752         rte_hash_free(tbl_rwc_test_param.h);
 753         return 0;
 754
 755 err:
 756         rte_eal_mp_wait_lcore();
 757         rte_hash_free(tbl_rwc_test_param.h);
 758         return -1;
 759 }
 760
 761 /*
 762  * Test lookup perf:
 763  * Reader(s) lookup keys absent in the table while
 764  * 'Main' thread adds with no key-shifts.
 765  */
 766 static int
 767 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 768                                 int htm, int ext_bkt)
 769 {
 770         unsigned int n, m;
 771         uint64_t i;
 772         int use_jhash = 0;
 773         uint8_t write_type = WRITE_NO_KEY_SHIFT;
 774         uint8_t read_type = READ_FAIL;
 775         int ret;
 776
 777         rte_atomic64_init(&greads);
 778         rte_atomic64_init(&gread_cycles);
 779
 780         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 781                 goto err;
 782         printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
 783         for (m = 0; m < 2; m++) {
 784                 if (m == 1) {
 785                         printf("\n** With bulk-lookup **\n");
 786                         read_type |= BULK_LOOKUP;
 787                 }
 788                 for (n = 0; n < NUM_TEST; n++) {
 789                         unsigned int tot_lcore = rte_lcore_count();
 790                         if (tot_lcore < rwc_core_cnt[n] + 1)
 791                                 goto finish;
 792
 793                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 794
 795                         rte_atomic64_clear(&greads);
 796                         rte_atomic64_clear(&gread_cycles);
 797
 798                         rte_hash_reset(tbl_rwc_test_param.h);
 799                         writer_done = 0;
 800
 801                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 802                                 rte_eal_remote_launch(test_rwc_reader,
 803                                                 (void *)(uintptr_t)read_type,
 804                                                         enabled_core_ids[i]);
 805                         ret = write_keys(write_type);
 806                         writer_done = 1;
 807
 808                         if (ret < 0)
 809                                 goto err;
 810                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 811                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 812                                         goto err;
 813
 814                         unsigned long long cycles_per_lookup =
 815                                 rte_atomic64_read(&gread_cycles) /
 816                                 rte_atomic64_read(&greads);
 817                         rwc_perf_results->w_no_ks_r_miss[m][n]
 818                                                 = cycles_per_lookup;
 819                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 820                 }
 821         }
 822
 823 finish:
 824         rte_hash_free(tbl_rwc_test_param.h);
 825         return 0;
 826
 827 err:
 828         rte_eal_mp_wait_lcore();
 829         rte_hash_free(tbl_rwc_test_param.h);
 830         return -1;
 831 }
 832
 833 /*
 834  * Test lookup perf:
 835  * Reader(s) lookup keys present in the table and not likely to be on the
 836  * shift path  while 'Main' thread adds keys causing key-shifts.
 837  */
 838 static int
 839 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 840                                     int rwc_lf, int htm, int ext_bkt)
 841 {
 842         unsigned int n, m;
 843         uint64_t i;
 844         int use_jhash = 0;
 845         int ret;
 846         uint8_t write_type;
 847         uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
 848
 849         rte_atomic64_init(&greads);
 850         rte_atomic64_init(&gread_cycles);
 851
 852         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 853                 goto err;
 854         printf("\nTest: Hash add - key shift, Hash lookup - hit"
 855                " (non-shift-path)\n");
 856         for (m = 0; m < 2; m++) {
 857                 if (m == 1) {
 858                         printf("\n** With bulk-lookup **\n");
 859                         read_type |= BULK_LOOKUP;
 860                 }
 861                 for (n = 0; n < NUM_TEST; n++) {
 862                         unsigned int tot_lcore = rte_lcore_count();
 863                         if (tot_lcore < rwc_core_cnt[n] + 1)
 864                                 goto finish;
 865
 866                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 867
 868                         rte_atomic64_clear(&greads);
 869                         rte_atomic64_clear(&gread_cycles);
 870
 871                         rte_hash_reset(tbl_rwc_test_param.h);
 872                         writer_done = 0;
 873                         write_type = WRITE_NO_KEY_SHIFT;
 874                         if (write_keys(write_type) < 0)
 875                                 goto err;
 876                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 877                                 rte_eal_remote_launch(test_rwc_reader,
 878                                                 (void *)(uintptr_t)read_type,
 879                                                         enabled_core_ids[i]);
 880                         write_type = WRITE_KEY_SHIFT;
 881                         ret = write_keys(write_type);
 882                         writer_done = 1;
 883
 884                         if (ret < 0)
 885                                 goto err;
 886                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 887                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 888                                         goto err;
 889
 890                         unsigned long long cycles_per_lookup =
 891                                 rte_atomic64_read(&gread_cycles) /
 892                                 rte_atomic64_read(&greads);
 893                         rwc_perf_results->w_ks_r_hit_nsp[m][n]
 894                                                 = cycles_per_lookup;
 895                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 896                 }
 897         }
 898
 899 finish:
 900         rte_hash_free(tbl_rwc_test_param.h);
 901         return 0;
 902
 903 err:
 904         rte_eal_mp_wait_lcore();
 905         rte_hash_free(tbl_rwc_test_param.h);
 906         return -1;
 907 }
 908
 909 /*
 910  * Test lookup perf:
 911  * Reader(s) lookup keys present in the table and likely on the shift-path while
 912  * 'Main' thread adds keys causing key-shifts.
 913  */
 914 static int
 915 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 916                                 int htm, int ext_bkt)
 917 {
 918         unsigned int n, m;
 919         uint64_t i;
 920         int use_jhash = 0;
 921         int ret;
 922         uint8_t write_type;
 923         uint8_t read_type = READ_PASS_SHIFT_PATH;
 924
 925         rte_atomic64_init(&greads);
 926         rte_atomic64_init(&gread_cycles);
 927
 928         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
 929                 goto err;
 930         printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
 931                "\n");
 932
 933         for (m = 0; m < 2; m++) {
 934                 if (m == 1) {
 935                         printf("\n** With bulk-lookup **\n");
 936                         read_type |= BULK_LOOKUP;
 937                 }
 938                 for (n = 0; n < NUM_TEST; n++) {
 939                         unsigned int tot_lcore = rte_lcore_count();
 940                         if (tot_lcore < rwc_core_cnt[n] + 1)
 941                                 goto finish;
 942
 943                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 944                         rte_atomic64_clear(&greads);
 945                         rte_atomic64_clear(&gread_cycles);
 946
 947                         rte_hash_reset(tbl_rwc_test_param.h);
 948                         writer_done = 0;
 949                         write_type = WRITE_NO_KEY_SHIFT;
 950                         if (write_keys(write_type) < 0)
 951                                 goto err;
 952                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 953                                 rte_eal_remote_launch(test_rwc_reader,
 954                                                 (void *)(uintptr_t)read_type,
 955                                                 enabled_core_ids[i]);
 956                         write_type = WRITE_KEY_SHIFT;
 957                         ret = write_keys(write_type);
 958                         writer_done = 1;
 959
 960                         if (ret < 0)
 961                                 goto err;
 962                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 963                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 964                                         goto err;
 965
 966                         unsigned long long cycles_per_lookup =
 967                                 rte_atomic64_read(&gread_cycles) /
 968                                 rte_atomic64_read(&greads);
 969                         rwc_perf_results->w_ks_r_hit_sp[m][n]
 970                                                 = cycles_per_lookup;
 971                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 972                 }
 973         }
 974
 975 finish:
 976         rte_hash_free(tbl_rwc_test_param.h);
 977         return 0;
 978
 979 err:
 980         rte_eal_mp_wait_lcore();
 981         rte_hash_free(tbl_rwc_test_param.h);
 982         return -1;
 983 }
 984
 985 /*
 986  * Test lookup perf:
 987  * Reader(s) lookup keys absent in the table while
 988  * 'Main' thread adds keys causing key-shifts.
 989  */
 990 static int
 991 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 992                              htm, int ext_bkt)
 993 {
 994         unsigned int n, m;
 995         uint64_t i;
 996         int use_jhash = 0;
 997         int ret;
 998         uint8_t write_type;
 999         uint8_t read_type = READ_FAIL;
1000
1001         rte_atomic64_init(&greads);
1002         rte_atomic64_init(&gread_cycles);
1003
1004         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1005                 goto err;
1006         printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
1007         for (m = 0; m < 2; m++) {
1008                 if (m == 1) {
1009                         printf("\n** With bulk-lookup **\n");
1010                         read_type |= BULK_LOOKUP;
1011                 }
1012                 for (n = 0; n < NUM_TEST; n++) {
1013                         unsigned int tot_lcore = rte_lcore_count();
1014                         if (tot_lcore < rwc_core_cnt[n] + 1)
1015                                 goto finish;
1016
1017                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1018
1019                         rte_atomic64_clear(&greads);
1020                         rte_atomic64_clear(&gread_cycles);
1021
1022                         rte_hash_reset(tbl_rwc_test_param.h);
1023                         writer_done = 0;
1024                         write_type = WRITE_NO_KEY_SHIFT;
1025                         if (write_keys(write_type) < 0)
1026                                 goto err;
1027                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1028                                 rte_eal_remote_launch(test_rwc_reader,
1029                                                 (void *)(uintptr_t)read_type,
1030                                                         enabled_core_ids[i]);
1031                         write_type = WRITE_KEY_SHIFT;
1032                         ret = write_keys(write_type);
1033                         writer_done = 1;
1034
1035                         if (ret < 0)
1036                                 goto err;
1037                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1038                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1039                                         goto err;
1040
1041                         unsigned long long cycles_per_lookup =
1042                                 rte_atomic64_read(&gread_cycles) /
1043                                 rte_atomic64_read(&greads);
1044                         rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
1045                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1046                 }
1047         }
1048
1049 finish:
1050         rte_hash_free(tbl_rwc_test_param.h);
1051         return 0;
1052
1053 err:
1054         rte_eal_mp_wait_lcore();
1055         rte_hash_free(tbl_rwc_test_param.h);
1056         return -1;
1057 }
1058
1059 /*
1060  * Test lookup perf for multi-writer:
1061  * Reader(s) lookup keys present in the table and likely on the shift-path while
1062  * Writers add keys causing key-shiftsi.
1063  * Writers are running in parallel, on different data plane cores.
1064  */
1065 static int
1066 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
1067                            int htm, int ext_bkt)
1068 {
1069         unsigned int n, m, k;
1070         uint64_t i;
1071         int use_jhash = 0;
1072         uint8_t write_type;
1073         uint8_t read_type = READ_PASS_SHIFT_PATH;
1074
1075         rte_atomic64_init(&greads);
1076         rte_atomic64_init(&gread_cycles);
1077
1078         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1079                 goto err;
1080         printf("\nTest: Multi-add-lookup\n");
1081         uint8_t pos_core;
1082         for (m = 1; m < NUM_TEST; m++) {
1083                 /* Calculate keys added by each writer */
1084                 tbl_rwc_test_param.single_insert =
1085                         tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
1086                 for (k = 0; k < 2; k++) {
1087                         if (k == 1) {
1088                                 printf("\n** With bulk-lookup **\n");
1089                                 read_type |= BULK_LOOKUP;
1090                         }
1091                         for (n = 0; n < NUM_TEST; n++) {
1092                                 unsigned int tot_lcore  = rte_lcore_count();
1093                                 if (tot_lcore < (rwc_core_cnt[n] +
1094                                      rwc_core_cnt[m] + 1))
1095                                         goto finish;
1096
1097                                 printf("\nNumber of writers: %u",
1098                                        rwc_core_cnt[m]);
1099                                 printf("\nNumber of readers: %u\n",
1100                                        rwc_core_cnt[n]);
1101
1102                                 rte_atomic64_clear(&greads);
1103                                 rte_atomic64_clear(&gread_cycles);
1104
1105                                 rte_hash_reset(tbl_rwc_test_param.h);
1106                                 writer_done = 0;
1107                                 write_type = WRITE_NO_KEY_SHIFT;
1108                                 if (write_keys(write_type) < 0)
1109                                         goto err;
1110
1111                                 /* Launch reader(s) */
1112                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1113                                         rte_eal_remote_launch(test_rwc_reader,
1114                                                 (void *)(uintptr_t)read_type,
1115                                                 enabled_core_ids[i]);
1116                                 write_type = WRITE_KEY_SHIFT;
1117                                 pos_core = 0;
1118
1119                                 /* Launch writers */
1120                                 for (; i <= rwc_core_cnt[m]
1121                                      + rwc_core_cnt[n]; i++) {
1122                                         rte_eal_remote_launch
1123                                                 (test_rwc_multi_writer,
1124                                                 (void *)(uintptr_t)pos_core,
1125                                                 enabled_core_ids[i]);
1126                                         pos_core++;
1127                                 }
1128
1129                                 /* Wait for writers to complete */
1130                                 for (i = rwc_core_cnt[n] + 1;
1131                                      i <= rwc_core_cnt[m] + rwc_core_cnt[n];
1132                                      i++)
1133                                         rte_eal_wait_lcore(enabled_core_ids[i]);
1134
1135                                 writer_done = 1;
1136
1137                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1138                                         if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1139                                                 goto err;
1140
1141                                 unsigned long long cycles_per_lookup =
1142                                         rte_atomic64_read(&gread_cycles)
1143                                         / rte_atomic64_read(&greads);
1144                                 rwc_perf_results->multi_rw[m][k][n]
1145                                         = cycles_per_lookup;
1146                                 printf("Cycles per lookup: %llu\n",
1147                                        cycles_per_lookup);
1148                         }
1149                 }
1150         }
1151
1152 finish:
1153         rte_hash_free(tbl_rwc_test_param.h);
1154         return 0;
1155
1156 err:
1157         rte_eal_mp_wait_lcore();
1158         rte_hash_free(tbl_rwc_test_param.h);
1159         return -1;
1160 }
1161
1162 /*
1163  * Test lookup perf:
1164  * Reader(s) lookup keys present in the extendable bkt.
1165  */
1166 static int
1167 test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
1168                                 int rwc_lf, int htm, int ext_bkt)
1169 {
1170         unsigned int n, m;
1171         uint64_t i;
1172         int use_jhash = 0;
1173         uint8_t write_type;
1174         uint8_t read_type = READ_PASS_KEY_SHIFTS_EXTBKT;
1175
1176         rte_atomic64_init(&greads);
1177         rte_atomic64_init(&gread_cycles);
1178
1179         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1180                 goto err;
1181         printf("\nTest: Hash add - key-shifts, read - hit (ext_bkt)\n");
1182         for (m = 0; m < 2; m++) {
1183                 if (m == 1) {
1184                         printf("\n** With bulk-lookup **\n");
1185                         read_type |= BULK_LOOKUP;
1186                 }
1187                 for (n = 0; n < NUM_TEST; n++) {
1188                         unsigned int tot_lcore = rte_lcore_count();
1189                         if (tot_lcore < rwc_core_cnt[n] + 1)
1190                                 goto finish;
1191
1192                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
1193
1194                         rte_atomic64_clear(&greads);
1195                         rte_atomic64_clear(&gread_cycles);
1196
1197                         rte_hash_reset(tbl_rwc_test_param.h);
1198                         write_type = WRITE_NO_KEY_SHIFT;
1199                         if (write_keys(write_type) < 0)
1200                                 goto err;
1201                         write_type = WRITE_KEY_SHIFT;
1202                         if (write_keys(write_type) < 0)
1203                                 goto err;
1204                         writer_done = 0;
1205                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1206                                 rte_eal_remote_launch(test_rwc_reader,
1207                                                 (void *)(uintptr_t)read_type,
1208                                                         enabled_core_ids[i]);
1209                         for (i = 0; i < tbl_rwc_test_param.count_keys_ks_extbkt;
1210                              i++) {
1211                                 if (rte_hash_del_key(tbl_rwc_test_param.h,
1212                                         tbl_rwc_test_param.keys_ks_extbkt + i)
1213                                                         < 0) {
1214                                         printf("Delete Failed: %u\n",
1215                                         tbl_rwc_test_param.keys_ks_extbkt[i]);
1216                                         goto err;
1217                                 }
1218                         }
1219                         writer_done = 1;
1220
1221                         for (i = 1; i <= rwc_core_cnt[n]; i++)
1222                                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
1223                                         goto err;
1224
1225                         unsigned long long cycles_per_lookup =
1226                                 rte_atomic64_read(&gread_cycles) /
1227                                 rte_atomic64_read(&greads);
1228                         rwc_perf_results->w_ks_r_hit_extbkt[m][n]
1229                                                 = cycles_per_lookup;
1230                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
1231                 }
1232         }
1233
1234 finish:
1235         rte_hash_free(tbl_rwc_test_param.h);
1236         return 0;
1237
1238 err:
1239         rte_eal_mp_wait_lcore();
1240         rte_hash_free(tbl_rwc_test_param.h);
1241         return -1;
1242 }
1243
1244 static int
1245 test_hash_readwrite_lf_main(void)
1246 {
1247         /*
1248          * Variables used to choose different tests.
1249          * rwc_lf indicates if read-write concurrency lock-free support is
1250          * enabled.
1251          * htm indicates if Hardware transactional memory support is enabled.
1252          */
1253         int rwc_lf = 0;
1254         int htm;
1255         int use_jhash = 0;
1256         int ext_bkt = 0;
1257
1258         if (rte_lcore_count() < 2) {
1259                 printf("Not enough cores for hash_readwrite_lf_autotest, expecting at least 2\n");
1260                 return TEST_SKIPPED;
1261         }
1262
1263         setlocale(LC_NUMERIC, "");
1264
1265         if (rte_tm_supported())
1266                 htm = 1;
1267         else
1268                 htm = 0;
1269
1270         if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
1271                 return -1;
1272         if (generate_keys() != 0)
1273                 return -1;
1274         if (get_enabled_cores_list() != 0)
1275                 return -1;
1276
1277         if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1278                 rwc_lf = 1;
1279                 ext_bkt = 1;
1280                 printf("Test lookup with read-write concurrency lock free support"
1281                        " enabled\n");
1282                 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1283                                                         htm, ext_bkt) < 0)
1284                         return -1;
1285                 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1286                                                         htm, ext_bkt) < 0)
1287                         return -1;
1288                 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1289                                                         htm, ext_bkt) < 0)
1290                         return -1;
1291                 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1292                                                         htm, ext_bkt) < 0)
1293                         return -1;
1294                 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm,
1295                                                  ext_bkt) < 0)
1296                         return -1;
1297                 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm,
1298                                                ext_bkt) < 0)
1299                         return -1;
1300                 if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
1301                                                         htm, ext_bkt) < 0)
1302                         return -1;
1303         }
1304         printf("\nTest lookup with read-write concurrency lock free support"
1305                " disabled\n");
1306         rwc_lf = 0;
1307         if (!htm) {
1308                 printf("With HTM Disabled\n");
1309                 if (!RUN_WITH_HTM_DISABLED) {
1310                         printf("Enable RUN_WITH_HTM_DISABLED to test with"
1311                                " lock-free disabled");
1312                         goto results;
1313                 }
1314         } else
1315                 printf("With HTM Enabled\n");
1316         if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm,
1317                                            ext_bkt) < 0)
1318                 return -1;
1319         if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1320                                                 ext_bkt) < 0)
1321                 return -1;
1322         if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1323                                                 htm, ext_bkt) < 0)
1324                 return -1;
1325         if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm,
1326                                                 ext_bkt) < 0)
1327                 return -1;
1328         if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm,
1329                                          ext_bkt) < 0)
1330                 return -1;
1331         if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm,
1332                                                         ext_bkt) < 0)
1333                 return -1;
1334         if (test_hash_add_ks_lookup_hit_extbkt(&rwc_non_lf_results, rwc_lf,
1335                                                 htm, ext_bkt) < 0)
1336                 return -1;
1337 results:
1338         printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1339         int i, j, k;
1340         for (j = 0; j < 2; j++) {
1341                 if (j == 1)
1342                         printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1343                                "**********#######\n\n");
1344                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1345                         "\t\t\t\t_________________\n");
1346                 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1347                        "\t\t\tCycles per lookup\n");
1348                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1349                        "\t\t\t_________________\n");
1350                 for (i = 0; i < NUM_TEST; i++) {
1351                         printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1352                         printf("Enabled\t\t");
1353                         printf("N/A\t\t");
1354                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1355                                 "%u\n\t\t\t\t\t\t\t\t",
1356                                 rwc_lf_results.w_no_ks_r_hit[j][i]);
1357                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1358                                 "%u\n\t\t\t\t\t\t\t\t",
1359                                 rwc_lf_results.w_no_ks_r_miss[j][i]);
1360                         printf("Hash add - key-shifts, lookup - hit"
1361                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1362                                rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1363                         printf("Hash add - key-shifts, lookup - hit "
1364                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1365                                rwc_lf_results.w_ks_r_hit_sp[j][i]);
1366                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1367                                 "%u\n\t\t\t\t\t\t\t\t",
1368                                 rwc_lf_results.w_ks_r_miss[j][i]);
1369                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1370                                 "%u\n\n\t\t\t\t",
1371                                 rwc_lf_results.w_ks_r_hit_extbkt[j][i]);
1372
1373                         printf("Disabled\t");
1374                         if (htm)
1375                                 printf("Enabled\t\t");
1376                         else
1377                                 printf("Disabled\t");
1378                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1379                                 "%u\n\t\t\t\t\t\t\t\t",
1380                                 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1381                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1382                                 "%u\n\t\t\t\t\t\t\t\t",
1383                                 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1384                         printf("Hash add - key-shifts, lookup - hit "
1385                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1386                                rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1387                         printf("Hash add - key-shifts, lookup - hit "
1388                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1389                                rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1390                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1391                                "%u\n\t\t\t\t\t\t\t\t",
1392                                rwc_non_lf_results.w_ks_r_miss[j][i]);
1393                         printf("Hash add - key-shifts, Hash lookup hit (ext_bkt)\t\t"
1394                                 "%u\n",
1395                                 rwc_non_lf_results.w_ks_r_hit_extbkt[j][i]);
1396
1397                         printf("_______\t\t_______\t\t_________\t___\t\t"
1398                                "_________\t\t\t\t\t\t_________________\n");
1399                 }
1400
1401                 for (i = 1; i < NUM_TEST; i++) {
1402                         for (k = 0; k < NUM_TEST; k++) {
1403                                 printf("%u", rwc_core_cnt[i]);
1404                                 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1405                                 printf("Enabled\t\t");
1406                                 printf("N/A\t\t");
1407                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1408                                        "\t\t",
1409                                        rwc_lf_results.multi_rw[i][j][k]);
1410                                 printf("Disabled\t");
1411                                 if (htm)
1412                                         printf("Enabled\t\t");
1413                                 else
1414                                         printf("Disabled\t");
1415                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1416                                        rwc_non_lf_results.multi_rw[i][j][k]);
1417
1418                                 printf("_______\t\t_______\t\t_________\t___"
1419                                        "\t\t_________\t\t\t\t\t\t"
1420                                        "_________________\n");
1421                         }
1422                 }
1423         }
1424         rte_free(tbl_rwc_test_param.keys);
1425         rte_free(tbl_rwc_test_param.keys_no_ks);
1426         rte_free(tbl_rwc_test_param.keys_ks);
1427         rte_free(tbl_rwc_test_param.keys_absent);
1428         rte_free(tbl_rwc_test_param.keys_shift_path);
1429         rte_free(scanned_bkts);
1430         return 0;
1431 }
1432
1433 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);