test/test/test_hash_readwrite.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #define RTE_RWTEST_FAIL 0
  20
  21 #define TOTAL_ENTRY (5*1024*1024)
  22 #define TOTAL_INSERT (4.5*1024*1024)
  23
  24 #define NUM_TEST 3
  25 unsigned int core_cnt[NUM_TEST] = {2, 4, 8};
  26
  27 unsigned int slave_core_ids[RTE_MAX_LCORE];
  28 struct perf {
  29         uint32_t single_read;
  30         uint32_t single_write;
  31         uint32_t read_only[NUM_TEST];
  32         uint32_t write_only[NUM_TEST];
  33         uint32_t read_write_r[NUM_TEST];
  34         uint32_t read_write_w[NUM_TEST];
  35 };
  36
  37 static struct perf htm_results, non_htm_results;
  38
  39 struct {
  40         uint32_t *keys;
  41         uint32_t *found;
  42         uint32_t num_insert;
  43         uint32_t rounded_tot_insert;
  44         struct rte_hash *h;
  45 } tbl_rw_test_param;
  46
  47 static rte_atomic64_t gcycles;
  48 static rte_atomic64_t ginsertions;
  49
  50 static rte_atomic64_t gread_cycles;
  51 static rte_atomic64_t gwrite_cycles;
  52
  53 static rte_atomic64_t greads;
  54 static rte_atomic64_t gwrites;
  55
  56 static int
  57 test_hash_readwrite_worker(__attribute__((unused)) void *arg)
  58 {
  59         uint64_t i, offset;
  60         uint32_t lcore_id = rte_lcore_id();
  61         uint64_t begin, cycles;
  62         int *ret;
  63
  64         ret = rte_malloc(NULL, sizeof(int) *
  65                                 tbl_rw_test_param.num_insert, 0);
  66         for (i = 0; i < rte_lcore_count(); i++) {
  67                 if (slave_core_ids[i] == lcore_id)
  68                         break;
  69         }
  70         offset = tbl_rw_test_param.num_insert * i;
  71
  72         printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n",
  73                lcore_id, tbl_rw_test_param.num_insert,
  74                offset, offset + tbl_rw_test_param.num_insert - 1);
  75
  76         begin = rte_rdtsc_precise();
  77
  78         for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
  79
  80                 if (rte_hash_lookup(tbl_rw_test_param.h,
  81                                 tbl_rw_test_param.keys + i) > 0)
  82                         break;
  83
  84                 ret[i - offset] = rte_hash_add_key(tbl_rw_test_param.h,
  85                                      tbl_rw_test_param.keys + i);
  86                 if (ret[i - offset] < 0)
  87                         break;
  88
  89                 /* lookup a random key */
  90                 uint32_t rand = rte_rand() % (i + 1 - offset);
  91
  92                 if (rte_hash_lookup(tbl_rw_test_param.h,
  93                                 tbl_rw_test_param.keys + rand) != ret[rand])
  94                         break;
  95
  96
  97                 if (rte_hash_del_key(tbl_rw_test_param.h,
  98                                 tbl_rw_test_param.keys + rand) != ret[rand])
  99                         break;
 100
 101                 ret[rand] = rte_hash_add_key(tbl_rw_test_param.h,
 102                                         tbl_rw_test_param.keys + rand);
 103                 if (ret[rand] < 0)
 104                         break;
 105
 106                 if (rte_hash_lookup(tbl_rw_test_param.h,
 107                         tbl_rw_test_param.keys + rand) != ret[rand])
 108                         break;
 109         }
 110
 111         cycles = rte_rdtsc_precise() - begin;
 112         rte_atomic64_add(&gcycles, cycles);
 113         rte_atomic64_add(&ginsertions, i - offset);
 114
 115         for (; i < offset + tbl_rw_test_param.num_insert; i++)
 116                 tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL;
 117
 118         rte_free(ret);
 119         return 0;
 120 }
 121
 122 static int
 123 init_params(int use_htm, int use_jhash)
 124 {
 125         unsigned int i;
 126
 127         uint32_t *keys = NULL;
 128         uint32_t *found = NULL;
 129         struct rte_hash *handle;
 130
 131         struct rte_hash_parameters hash_params = {
 132                 .entries = TOTAL_ENTRY,
 133                 .key_len = sizeof(uint32_t),
 134                 .hash_func_init_val = 0,
 135                 .socket_id = rte_socket_id(),
 136         };
 137         if (use_jhash)
 138                 hash_params.hash_func = rte_jhash;
 139         else
 140                 hash_params.hash_func = rte_hash_crc;
 141
 142         if (use_htm)
 143                 hash_params.extra_flag =
 144                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 145                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 146                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 147         else
 148                 hash_params.extra_flag =
 149                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 150                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 151
 152         hash_params.name = "tests";
 153
 154         handle = rte_hash_create(&hash_params);
 155         if (handle == NULL) {
 156                 printf("hash creation failed");
 157                 return -1;
 158         }
 159
 160         tbl_rw_test_param.h = handle;
 161         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
 162
 163         if (keys == NULL) {
 164                 printf("RTE_MALLOC failed\n");
 165                 goto err;
 166         }
 167
 168         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
 169         if (found == NULL) {
 170                 printf("RTE_ZMALLOC failed\n");
 171                 goto err;
 172         }
 173
 174         tbl_rw_test_param.keys = keys;
 175         tbl_rw_test_param.found = found;
 176
 177         for (i = 0; i < TOTAL_ENTRY; i++)
 178                 keys[i] = i;
 179
 180         return 0;
 181
 182 err:
 183         rte_free(keys);
 184         rte_hash_free(handle);
 185
 186         return -1;
 187 }
 188
 189 static int
 190 test_hash_readwrite_functional(int use_htm)
 191 {
 192         unsigned int i;
 193         const void *next_key;
 194         void *next_data;
 195         uint32_t iter = 0;
 196
 197         uint32_t duplicated_keys = 0;
 198         uint32_t lost_keys = 0;
 199         int use_jhash = 1;
 200         int slave_cnt = rte_lcore_count() - 1;
 201
 202         rte_atomic64_init(&gcycles);
 203         rte_atomic64_clear(&gcycles);
 204
 205         rte_atomic64_init(&ginsertions);
 206         rte_atomic64_clear(&ginsertions);
 207
 208         if (init_params(use_htm, use_jhash) != 0)
 209                 goto err;
 210
 211         tbl_rw_test_param.num_insert =
 212                 TOTAL_INSERT / slave_cnt;
 213
 214         tbl_rw_test_param.rounded_tot_insert =
 215                 tbl_rw_test_param.num_insert
 216                 * slave_cnt;
 217
 218         printf("++++++++Start function tests:+++++++++\n");
 219
 220         /* Fire all threads. */
 221         rte_eal_mp_remote_launch(test_hash_readwrite_worker,
 222                                  NULL, SKIP_MASTER);
 223         rte_eal_mp_wait_lcore();
 224
 225         while (rte_hash_iterate(tbl_rw_test_param.h, &next_key,
 226                         &next_data, &iter) >= 0) {
 227                 /* Search for the key in the list of keys added .*/
 228                 i = *(const uint32_t *)next_key;
 229                 tbl_rw_test_param.found[i]++;
 230         }
 231
 232         for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
 233                 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
 234                         if (tbl_rw_test_param.found[i] > 1) {
 235                                 duplicated_keys++;
 236                                 break;
 237                         }
 238                         if (tbl_rw_test_param.found[i] == 0) {
 239                                 lost_keys++;
 240                                 printf("key %d is lost\n", i);
 241                                 break;
 242                         }
 243                 }
 244         }
 245
 246         if (duplicated_keys > 0) {
 247                 printf("%d key duplicated\n", duplicated_keys);
 248                 goto err_free;
 249         }
 250
 251         if (lost_keys > 0) {
 252                 printf("%d key lost\n", lost_keys);
 253                 goto err_free;
 254         }
 255
 256         printf("No key corrupted during read-write test.\n");
 257
 258         unsigned long long int cycles_per_insertion =
 259                 rte_atomic64_read(&gcycles) /
 260                 rte_atomic64_read(&ginsertions);
 261
 262         printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion);
 263
 264         rte_free(tbl_rw_test_param.found);
 265         rte_free(tbl_rw_test_param.keys);
 266         rte_hash_free(tbl_rw_test_param.h);
 267         printf("+++++++++Complete function tests+++++++++\n");
 268         return 0;
 269
 270 err_free:
 271         rte_free(tbl_rw_test_param.found);
 272         rte_free(tbl_rw_test_param.keys);
 273         rte_hash_free(tbl_rw_test_param.h);
 274 err:
 275         return -1;
 276 }
 277
 278 static int
 279 test_rw_reader(void *arg)
 280 {
 281         uint64_t i;
 282         uint64_t begin, cycles;
 283         uint64_t read_cnt = (uint64_t)((uintptr_t)arg);
 284
 285         begin = rte_rdtsc_precise();
 286         for (i = 0; i < read_cnt; i++) {
 287                 void *data;
 288                 rte_hash_lookup_data(tbl_rw_test_param.h,
 289                                 tbl_rw_test_param.keys + i,
 290                                 &data);
 291                 if (i != (uint64_t)(uintptr_t)data) {
 292                         printf("lookup find wrong value %"PRIu64","
 293                                 "%"PRIu64"\n", i,
 294                                 (uint64_t)(uintptr_t)data);
 295                         break;
 296                 }
 297         }
 298
 299         cycles = rte_rdtsc_precise() - begin;
 300         rte_atomic64_add(&gread_cycles, cycles);
 301         rte_atomic64_add(&greads, i);
 302         return 0;
 303 }
 304
 305 static int
 306 test_rw_writer(void *arg)
 307 {
 308         uint64_t i;
 309         uint32_t lcore_id = rte_lcore_id();
 310         uint64_t begin, cycles;
 311         int ret;
 312         uint64_t start_coreid = (uint64_t)(uintptr_t)arg;
 313         uint64_t offset;
 314
 315         for (i = 0; i < rte_lcore_count(); i++) {
 316                 if (slave_core_ids[i] == lcore_id)
 317                         break;
 318         }
 319
 320         offset = TOTAL_INSERT / 2 + (i - (start_coreid)) *
 321                                 tbl_rw_test_param.num_insert;
 322         begin = rte_rdtsc_precise();
 323         for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
 324                 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 325                                 tbl_rw_test_param.keys + i,
 326                                 (void *)((uintptr_t)i));
 327                 if (ret < 0) {
 328                         printf("writer failed %"PRIu64"\n", i);
 329                         break;
 330                 }
 331         }
 332
 333         cycles = rte_rdtsc_precise() - begin;
 334         rte_atomic64_add(&gwrite_cycles, cycles);
 335         rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert);
 336         return 0;
 337 }
 338
 339 static int
 340 test_hash_readwrite_perf(struct perf *perf_results, int use_htm,
 341                                                         int reader_faster)
 342 {
 343         unsigned int n;
 344         int ret;
 345         int start_coreid;
 346         uint64_t i, read_cnt;
 347
 348         const void *next_key;
 349         void *next_data;
 350         uint32_t iter = 0;
 351         int use_jhash = 0;
 352
 353         uint32_t duplicated_keys = 0;
 354         uint32_t lost_keys = 0;
 355
 356         uint64_t start = 0, end = 0;
 357
 358         rte_atomic64_init(&greads);
 359         rte_atomic64_init(&gwrites);
 360         rte_atomic64_clear(&gwrites);
 361         rte_atomic64_clear(&greads);
 362
 363         rte_atomic64_init(&gread_cycles);
 364         rte_atomic64_clear(&gread_cycles);
 365         rte_atomic64_init(&gwrite_cycles);
 366         rte_atomic64_clear(&gwrite_cycles);
 367
 368         if (init_params(use_htm, use_jhash) != 0)
 369                 goto err;
 370
 371         /*
 372          * Do a readers finish faster or writers finish faster test.
 373          * When readers finish faster, we timing the readers, and when writers
 374          * finish faster, we timing the writers.
 375          * Divided by 10 or 2 is just experimental values to vary the workload
 376          * of readers.
 377          */
 378         if (reader_faster) {
 379                 printf("++++++Start perf test: reader++++++++\n");
 380                 read_cnt = TOTAL_INSERT / 10;
 381         } else {
 382                 printf("++++++Start perf test: writer++++++++\n");
 383                 read_cnt = TOTAL_INSERT / 2;
 384         }
 385
 386         /* We first test single thread performance */
 387         start = rte_rdtsc_precise();
 388         /* Insert half of the keys */
 389         for (i = 0; i < TOTAL_INSERT / 2; i++) {
 390                 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 391                                      tbl_rw_test_param.keys + i,
 392                                         (void *)((uintptr_t)i));
 393                 if (ret < 0) {
 394                         printf("Failed to insert half of keys\n");
 395                         goto err_free;
 396                 }
 397         }
 398         end = rte_rdtsc_precise() - start;
 399         perf_results->single_write = end / i;
 400
 401         start = rte_rdtsc_precise();
 402
 403         for (i = 0; i < read_cnt; i++) {
 404                 void *data;
 405                 rte_hash_lookup_data(tbl_rw_test_param.h,
 406                                 tbl_rw_test_param.keys + i,
 407                                 &data);
 408                 if (i != (uint64_t)(uintptr_t)data) {
 409                         printf("lookup find wrong value"
 410                                         " %"PRIu64",%"PRIu64"\n", i,
 411                                         (uint64_t)(uintptr_t)data);
 412                         break;
 413                 }
 414         }
 415         end = rte_rdtsc_precise() - start;
 416         perf_results->single_read = end / i;
 417
 418         for (n = 0; n < NUM_TEST; n++) {
 419                 unsigned int tot_slave_lcore = rte_lcore_count() - 1;
 420                 if (tot_slave_lcore < core_cnt[n] * 2)
 421                         goto finish;
 422
 423                 rte_atomic64_clear(&greads);
 424                 rte_atomic64_clear(&gread_cycles);
 425                 rte_atomic64_clear(&gwrites);
 426                 rte_atomic64_clear(&gwrite_cycles);
 427
 428                 rte_hash_reset(tbl_rw_test_param.h);
 429
 430                 tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n];
 431                 tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 +
 432                                                 tbl_rw_test_param.num_insert *
 433                                                 core_cnt[n];
 434
 435                 for (i = 0; i < TOTAL_INSERT / 2; i++) {
 436                         ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 437                                         tbl_rw_test_param.keys + i,
 438                                         (void *)((uintptr_t)i));
 439                         if (ret < 0) {
 440                                 printf("Failed to insert half of keys\n");
 441                                 goto err_free;
 442                         }
 443                 }
 444
 445                 /* Then test multiple thread case but only all reads or
 446                  * all writes
 447                  */
 448
 449                 /* Test only reader cases */
 450                 for (i = 0; i < core_cnt[n]; i++)
 451                         rte_eal_remote_launch(test_rw_reader,
 452                                         (void *)(uintptr_t)read_cnt,
 453                                         slave_core_ids[i]);
 454
 455                 rte_eal_mp_wait_lcore();
 456
 457                 start_coreid = i;
 458                 /* Test only writer cases */
 459                 for (; i < core_cnt[n] * 2; i++)
 460                         rte_eal_remote_launch(test_rw_writer,
 461                                         (void *)((uintptr_t)start_coreid),
 462                                         slave_core_ids[i]);
 463
 464                 rte_eal_mp_wait_lcore();
 465
 466                 if (reader_faster) {
 467                         unsigned long long int cycles_per_insertion =
 468                                 rte_atomic64_read(&gread_cycles) /
 469                                 rte_atomic64_read(&greads);
 470                         perf_results->read_only[n] = cycles_per_insertion;
 471                         printf("Reader only: cycles per lookup: %llu\n",
 472                                                         cycles_per_insertion);
 473                 }
 474
 475                 else {
 476                         unsigned long long int cycles_per_insertion =
 477                                 rte_atomic64_read(&gwrite_cycles) /
 478                                 rte_atomic64_read(&gwrites);
 479                         perf_results->write_only[n] = cycles_per_insertion;
 480                         printf("Writer only: cycles per writes: %llu\n",
 481                                                         cycles_per_insertion);
 482                 }
 483
 484                 rte_atomic64_clear(&greads);
 485                 rte_atomic64_clear(&gread_cycles);
 486                 rte_atomic64_clear(&gwrites);
 487                 rte_atomic64_clear(&gwrite_cycles);
 488
 489                 rte_hash_reset(tbl_rw_test_param.h);
 490
 491                 for (i = 0; i < TOTAL_INSERT / 2; i++) {
 492                         ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 493                                         tbl_rw_test_param.keys + i,
 494                                         (void *)((uintptr_t)i));
 495                         if (ret < 0) {
 496                                 printf("Failed to insert half of keys\n");
 497                                 goto err_free;
 498                         }
 499                 }
 500
 501                 start_coreid = core_cnt[n];
 502
 503                 if (reader_faster) {
 504                         for (i = core_cnt[n]; i < core_cnt[n] * 2; i++)
 505                                 rte_eal_remote_launch(test_rw_writer,
 506                                         (void *)((uintptr_t)start_coreid),
 507                                         slave_core_ids[i]);
 508                         for (i = 0; i < core_cnt[n]; i++)
 509                                 rte_eal_remote_launch(test_rw_reader,
 510                                         (void *)(uintptr_t)read_cnt,
 511                                         slave_core_ids[i]);
 512                 } else {
 513                         for (i = 0; i < core_cnt[n]; i++)
 514                                 rte_eal_remote_launch(test_rw_reader,
 515                                         (void *)(uintptr_t)read_cnt,
 516                                         slave_core_ids[i]);
 517                         for (; i < core_cnt[n] * 2; i++)
 518                                 rte_eal_remote_launch(test_rw_writer,
 519                                         (void *)((uintptr_t)start_coreid),
 520                                         slave_core_ids[i]);
 521                 }
 522
 523                 rte_eal_mp_wait_lcore();
 524
 525                 while (rte_hash_iterate(tbl_rw_test_param.h,
 526                                 &next_key, &next_data, &iter) >= 0) {
 527                         /* Search for the key in the list of keys added .*/
 528                         i = *(const uint32_t *)next_key;
 529                         tbl_rw_test_param.found[i]++;
 530                 }
 531
 532                 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
 533                         if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
 534                                 if (tbl_rw_test_param.found[i] > 1) {
 535                                         duplicated_keys++;
 536                                         break;
 537                                 }
 538                                 if (tbl_rw_test_param.found[i] == 0) {
 539                                         lost_keys++;
 540                                         printf("key %"PRIu64" is lost\n", i);
 541                                         break;
 542                                 }
 543                         }
 544                 }
 545
 546                 if (duplicated_keys > 0) {
 547                         printf("%d key duplicated\n", duplicated_keys);
 548                         goto err_free;
 549                 }
 550
 551                 if (lost_keys > 0) {
 552                         printf("%d key lost\n", lost_keys);
 553                         goto err_free;
 554                 }
 555
 556                 printf("No key corrupted during read-write test.\n");
 557
 558                 if (reader_faster) {
 559                         unsigned long long int cycles_per_insertion =
 560                                 rte_atomic64_read(&gread_cycles) /
 561                                 rte_atomic64_read(&greads);
 562                         perf_results->read_write_r[n] = cycles_per_insertion;
 563                         printf("Read-write cycles per lookup: %llu\n",
 564                                                         cycles_per_insertion);
 565                 }
 566
 567                 else {
 568                         unsigned long long int cycles_per_insertion =
 569                                 rte_atomic64_read(&gwrite_cycles) /
 570                                 rte_atomic64_read(&gwrites);
 571                         perf_results->read_write_w[n] = cycles_per_insertion;
 572                         printf("Read-write cycles per writes: %llu\n",
 573                                                         cycles_per_insertion);
 574                 }
 575         }
 576
 577 finish:
 578         rte_free(tbl_rw_test_param.found);
 579         rte_free(tbl_rw_test_param.keys);
 580         rte_hash_free(tbl_rw_test_param.h);
 581         return 0;
 582
 583 err_free:
 584         rte_free(tbl_rw_test_param.found);
 585         rte_free(tbl_rw_test_param.keys);
 586         rte_hash_free(tbl_rw_test_param.h);
 587
 588 err:
 589         return -1;
 590 }
 591
 592 static int
 593 test_hash_readwrite_main(void)
 594 {
 595         /*
 596          * Variables used to choose different tests.
 597          * use_htm indicates if hardware transactional memory should be used.
 598          * reader_faster indicates if the reader threads should finish earlier
 599          * than writer threads. This is to timing either reader threads or
 600          * writer threads for performance numbers.
 601          */
 602         int use_htm, reader_faster;
 603         unsigned int i = 0, core_id = 0;
 604
 605         if (rte_lcore_count() <= 2) {
 606                 printf("More than two lcores are required "
 607                         "to do read write test\n");
 608                 return 0;
 609         }
 610
 611         RTE_LCORE_FOREACH_SLAVE(core_id) {
 612                 slave_core_ids[i] = core_id;
 613                 i++;
 614         }
 615
 616         setlocale(LC_NUMERIC, "");
 617
 618         if (rte_tm_supported()) {
 619                 printf("Hardware transactional memory (lock elision) "
 620                         "is supported\n");
 621
 622                 printf("Test read-write with Hardware transactional memory\n");
 623
 624                 use_htm = 1;
 625                 if (test_hash_readwrite_functional(use_htm) < 0)
 626                         return -1;
 627
 628                 reader_faster = 1;
 629                 if (test_hash_readwrite_perf(&htm_results, use_htm,
 630                                                         reader_faster) < 0)
 631                         return -1;
 632
 633                 reader_faster = 0;
 634                 if (test_hash_readwrite_perf(&htm_results, use_htm,
 635                                                         reader_faster) < 0)
 636                         return -1;
 637         } else {
 638                 printf("Hardware transactional memory (lock elision) "
 639                         "is NOT supported\n");
 640         }
 641
 642         printf("Test read-write without Hardware transactional memory\n");
 643         use_htm = 0;
 644         if (test_hash_readwrite_functional(use_htm) < 0)
 645                 return -1;
 646         reader_faster = 1;
 647         if (test_hash_readwrite_perf(&non_htm_results, use_htm,
 648                                                         reader_faster) < 0)
 649                 return -1;
 650         reader_faster = 0;
 651         if (test_hash_readwrite_perf(&non_htm_results, use_htm,
 652                                                         reader_faster) < 0)
 653                 return -1;
 654
 655         printf("Results summary:\n");
 656
 657         printf("single read: %u\n", htm_results.single_read);
 658         printf("single write: %u\n", htm_results.single_write);
 659         for (i = 0; i < NUM_TEST; i++) {
 660                 printf("core_cnt: %u\n", core_cnt[i]);
 661                 printf("HTM:\n");
 662                 printf("read only: %u\n", htm_results.read_only[i]);
 663                 printf("write only: %u\n", htm_results.write_only[i]);
 664                 printf("read-write read: %u\n", htm_results.read_write_r[i]);
 665                 printf("read-write write: %u\n", htm_results.read_write_w[i]);
 666
 667                 printf("non HTM:\n");
 668                 printf("read only: %u\n", non_htm_results.read_only[i]);
 669                 printf("write only: %u\n", non_htm_results.write_only[i]);
 670                 printf("read-write read: %u\n",
 671                         non_htm_results.read_write_r[i]);
 672                 printf("read-write write: %u\n",
 673                         non_htm_results.read_write_w[i]);
 674         }
 675
 676         return 0;
 677 }
 678
 679 REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main);