app/test/test_rcu_qsbr_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2018 Arm Limited
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdbool.h>
   7 #include <inttypes.h>
   8 #include <rte_pause.h>
   9 #include <rte_rcu_qsbr.h>
  10 #include <rte_hash.h>
  11 #include <rte_hash_crc.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <unistd.h>
  15
  16 #include "test.h"
  17
  18 /* Check condition and return an error if true. */
  19 #define TEST_RCU_MAX_LCORE 128
  20 static uint16_t enabled_core_ids[TEST_RCU_MAX_LCORE];
  21 static uint8_t num_cores;
  22
  23 static uint32_t *keys;
  24 #define TOTAL_ENTRY (1024 * 8)
  25 #define COUNTER_VALUE 4096
  26 static uint32_t *hash_data[TOTAL_ENTRY];
  27 static volatile uint8_t writer_done;
  28 static volatile uint8_t all_registered;
  29 static volatile uint32_t thr_id;
  30
  31 static struct rte_rcu_qsbr *t[TEST_RCU_MAX_LCORE];
  32 static struct rte_hash *h;
  33 static char hash_name[8];
  34 static rte_atomic64_t updates, checks;
  35 static rte_atomic64_t update_cycles, check_cycles;
  36
  37 /* Scale down results to 1000 operations to support lower
  38  * granularity clocks.
  39  */
  40 #define RCU_SCALE_DOWN 1000
  41
  42 /* Simple way to allocate thread ids in 0 to TEST_RCU_MAX_LCORE space */
  43 static inline uint32_t
  44 alloc_thread_id(void)
  45 {
  46         uint32_t tmp_thr_id;
  47
  48         tmp_thr_id = __atomic_fetch_add(&thr_id, 1, __ATOMIC_RELAXED);
  49         if (tmp_thr_id >= TEST_RCU_MAX_LCORE)
  50                 printf("Invalid thread id %u\n", tmp_thr_id);
  51
  52         return tmp_thr_id;
  53 }
  54
  55 static inline int
  56 get_enabled_cores_mask(void)
  57 {
  58         uint16_t core_id;
  59         uint32_t max_cores = rte_lcore_count();
  60
  61         if (max_cores > TEST_RCU_MAX_LCORE) {
  62                 printf("Number of cores exceed %d\n", TEST_RCU_MAX_LCORE);
  63                 return -1;
  64         }
  65
  66         core_id = 0;
  67         num_cores = 0;
  68         RTE_LCORE_FOREACH_SLAVE(core_id) {
  69                 enabled_core_ids[num_cores] = core_id;
  70                 num_cores++;
  71         }
  72
  73         return 0;
  74 }
  75
  76 static int
  77 test_rcu_qsbr_reader_perf(void *arg)
  78 {
  79         bool writer_present = (bool)arg;
  80         uint32_t thread_id = alloc_thread_id();
  81         uint64_t loop_cnt = 0;
  82         uint64_t begin, cycles;
  83
  84         /* Register for report QS */
  85         rte_rcu_qsbr_thread_register(t[0], thread_id);
  86         /* Make the thread online */
  87         rte_rcu_qsbr_thread_online(t[0], thread_id);
  88
  89         begin = rte_rdtsc_precise();
  90
  91         if (writer_present) {
  92                 while (!writer_done) {
  93                         /* Update quiescent state counter */
  94                         rte_rcu_qsbr_quiescent(t[0], thread_id);
  95                         loop_cnt++;
  96                 }
  97         } else {
  98                 while (loop_cnt < 100000000) {
  99                         /* Update quiescent state counter */
 100                         rte_rcu_qsbr_quiescent(t[0], thread_id);
 101                         loop_cnt++;
 102                 }
 103         }
 104
 105         cycles = rte_rdtsc_precise() - begin;
 106         rte_atomic64_add(&update_cycles, cycles);
 107         rte_atomic64_add(&updates, loop_cnt);
 108
 109         /* Make the thread offline */
 110         rte_rcu_qsbr_thread_offline(t[0], thread_id);
 111         /* Unregister before exiting to avoid writer from waiting */
 112         rte_rcu_qsbr_thread_unregister(t[0], thread_id);
 113
 114         return 0;
 115 }
 116
 117 static int
 118 test_rcu_qsbr_writer_perf(void *arg)
 119 {
 120         bool wait = (bool)arg;
 121         uint64_t token = 0;
 122         uint64_t loop_cnt = 0;
 123         uint64_t begin, cycles;
 124
 125         begin = rte_rdtsc_precise();
 126
 127         do {
 128                 /* Start the quiescent state query process */
 129                 if (wait)
 130                         token = rte_rcu_qsbr_start(t[0]);
 131
 132                 /* Check quiescent state status */
 133                 rte_rcu_qsbr_check(t[0], token, wait);
 134                 loop_cnt++;
 135         } while (loop_cnt < 20000000);
 136
 137         cycles = rte_rdtsc_precise() - begin;
 138         rte_atomic64_add(&check_cycles, cycles);
 139         rte_atomic64_add(&checks, loop_cnt);
 140         return 0;
 141 }
 142
 143 /*
 144  * Perf test: Reader/writer
 145  * Single writer, Multiple Readers, Single QS var, Non-Blocking rcu_qsbr_check
 146  */
 147 static int
 148 test_rcu_qsbr_perf(void)
 149 {
 150         int i, sz;
 151         int tmp_num_cores;
 152
 153         writer_done = 0;
 154
 155         rte_atomic64_clear(&updates);
 156         rte_atomic64_clear(&update_cycles);
 157         rte_atomic64_clear(&checks);
 158         rte_atomic64_clear(&check_cycles);
 159
 160         printf("\nPerf Test: %d Readers/1 Writer('wait' in qsbr_check == true)\n",
 161                 num_cores - 1);
 162
 163         __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 164
 165         if (all_registered == 1)
 166                 tmp_num_cores = num_cores - 1;
 167         else
 168                 tmp_num_cores = TEST_RCU_MAX_LCORE;
 169
 170         sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
 171         t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
 172                                                 RTE_CACHE_LINE_SIZE);
 173         /* QS variable is initialized */
 174         rte_rcu_qsbr_init(t[0], tmp_num_cores);
 175
 176         /* Reader threads are launched */
 177         for (i = 0; i < num_cores - 1; i++)
 178                 rte_eal_remote_launch(test_rcu_qsbr_reader_perf, (void *)1,
 179                                         enabled_core_ids[i]);
 180
 181         /* Writer thread is launched */
 182         rte_eal_remote_launch(test_rcu_qsbr_writer_perf,
 183                               (void *)1, enabled_core_ids[i]);
 184
 185         /* Wait for the writer thread */
 186         rte_eal_wait_lcore(enabled_core_ids[i]);
 187         writer_done = 1;
 188
 189         /* Wait until all readers have exited */
 190         rte_eal_mp_wait_lcore();
 191
 192         printf("Total RCU updates = %"PRIi64"\n", rte_atomic64_read(&updates));
 193         printf("Cycles per %d updates: %"PRIi64"\n", RCU_SCALE_DOWN,
 194                 rte_atomic64_read(&update_cycles) /
 195                 (rte_atomic64_read(&updates) / RCU_SCALE_DOWN));
 196         printf("Total RCU checks = %"PRIi64"\n", rte_atomic64_read(&checks));
 197         printf("Cycles per %d checks: %"PRIi64"\n", RCU_SCALE_DOWN,
 198                 rte_atomic64_read(&check_cycles) /
 199                 (rte_atomic64_read(&checks) / RCU_SCALE_DOWN));
 200
 201         rte_free(t[0]);
 202
 203         return 0;
 204 }
 205
 206 /*
 207  * Perf test: Readers
 208  * Single writer, Multiple readers, Single QS variable
 209  */
 210 static int
 211 test_rcu_qsbr_rperf(void)
 212 {
 213         int i, sz;
 214         int tmp_num_cores;
 215
 216         rte_atomic64_clear(&updates);
 217         rte_atomic64_clear(&update_cycles);
 218
 219         __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 220
 221         printf("\nPerf Test: %d Readers\n", num_cores);
 222
 223         if (all_registered == 1)
 224                 tmp_num_cores = num_cores;
 225         else
 226                 tmp_num_cores = TEST_RCU_MAX_LCORE;
 227
 228         sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
 229         t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
 230                                                 RTE_CACHE_LINE_SIZE);
 231         /* QS variable is initialized */
 232         rte_rcu_qsbr_init(t[0], tmp_num_cores);
 233
 234         /* Reader threads are launched */
 235         for (i = 0; i < num_cores; i++)
 236                 rte_eal_remote_launch(test_rcu_qsbr_reader_perf, NULL,
 237                                         enabled_core_ids[i]);
 238
 239         /* Wait until all readers have exited */
 240         rte_eal_mp_wait_lcore();
 241
 242         printf("Total RCU updates = %"PRIi64"\n", rte_atomic64_read(&updates));
 243         printf("Cycles per %d updates: %"PRIi64"\n", RCU_SCALE_DOWN,
 244                 rte_atomic64_read(&update_cycles) /
 245                 (rte_atomic64_read(&updates) / RCU_SCALE_DOWN));
 246
 247         rte_free(t[0]);
 248
 249         return 0;
 250 }
 251
 252 /*
 253  * Perf test:
 254  * Multiple writer, Single QS variable, Non-blocking rcu_qsbr_check
 255  */
 256 static int
 257 test_rcu_qsbr_wperf(void)
 258 {
 259         int i, sz;
 260
 261         rte_atomic64_clear(&checks);
 262         rte_atomic64_clear(&check_cycles);
 263
 264         __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 265
 266         printf("\nPerf test: %d Writers ('wait' in qsbr_check == false)\n",
 267                 num_cores);
 268
 269         /* Number of readers does not matter for QS variable in this test
 270          * case as no reader will be registered.
 271          */
 272         sz = rte_rcu_qsbr_get_memsize(TEST_RCU_MAX_LCORE);
 273         t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
 274                                                 RTE_CACHE_LINE_SIZE);
 275         /* QS variable is initialized */
 276         rte_rcu_qsbr_init(t[0], TEST_RCU_MAX_LCORE);
 277
 278         /* Writer threads are launched */
 279         for (i = 0; i < num_cores; i++)
 280                 rte_eal_remote_launch(test_rcu_qsbr_writer_perf,
 281                                 (void *)0, enabled_core_ids[i]);
 282
 283         /* Wait until all readers have exited */
 284         rte_eal_mp_wait_lcore();
 285
 286         printf("Total RCU checks = %"PRIi64"\n", rte_atomic64_read(&checks));
 287         printf("Cycles per %d checks: %"PRIi64"\n", RCU_SCALE_DOWN,
 288                 rte_atomic64_read(&check_cycles) /
 289                 (rte_atomic64_read(&checks) / RCU_SCALE_DOWN));
 290
 291         rte_free(t[0]);
 292
 293         return 0;
 294 }
 295
 296 /*
 297  * RCU test cases using rte_hash data structure.
 298  */
 299 static int
 300 test_rcu_qsbr_hash_reader(void *arg)
 301 {
 302         struct rte_rcu_qsbr *temp;
 303         struct rte_hash *hash = NULL;
 304         int i;
 305         uint64_t loop_cnt = 0;
 306         uint64_t begin, cycles;
 307         uint32_t thread_id = alloc_thread_id();
 308         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 309         uint32_t *pdata;
 310
 311         temp = t[read_type];
 312         hash = h;
 313
 314         rte_rcu_qsbr_thread_register(temp, thread_id);
 315
 316         begin = rte_rdtsc_precise();
 317
 318         do {
 319                 rte_rcu_qsbr_thread_online(temp, thread_id);
 320                 for (i = 0; i < TOTAL_ENTRY; i++) {
 321                         rte_rcu_qsbr_lock(temp, thread_id);
 322                         if (rte_hash_lookup_data(hash, keys + i,
 323                                         (void **)&pdata) != -ENOENT) {
 324                                 pdata[thread_id] = 0;
 325                                 while (pdata[thread_id] < COUNTER_VALUE)
 326                                         pdata[thread_id]++;
 327                         }
 328                         rte_rcu_qsbr_unlock(temp, thread_id);
 329                 }
 330                 /* Update quiescent state counter */
 331                 rte_rcu_qsbr_quiescent(temp, thread_id);
 332                 rte_rcu_qsbr_thread_offline(temp, thread_id);
 333                 loop_cnt++;
 334         } while (!writer_done);
 335
 336         cycles = rte_rdtsc_precise() - begin;
 337         rte_atomic64_add(&update_cycles, cycles);
 338         rte_atomic64_add(&updates, loop_cnt);
 339
 340         rte_rcu_qsbr_thread_unregister(temp, thread_id);
 341
 342         return 0;
 343 }
 344
 345 static struct rte_hash *init_hash(void)
 346 {
 347         int i;
 348         struct rte_hash *hash = NULL;
 349
 350         snprintf(hash_name, 8, "hash");
 351         struct rte_hash_parameters hash_params = {
 352                 .entries = TOTAL_ENTRY,
 353                 .key_len = sizeof(uint32_t),
 354                 .hash_func_init_val = 0,
 355                 .socket_id = rte_socket_id(),
 356                 .hash_func = rte_hash_crc,
 357                 .extra_flag =
 358                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF,
 359                 .name = hash_name,
 360         };
 361
 362         hash = rte_hash_create(&hash_params);
 363         if (hash == NULL) {
 364                 printf("Hash create Failed\n");
 365                 return NULL;
 366         }
 367
 368         for (i = 0; i < TOTAL_ENTRY; i++) {
 369                 hash_data[i] = rte_zmalloc(NULL,
 370                                 sizeof(uint32_t) * TEST_RCU_MAX_LCORE, 0);
 371                 if (hash_data[i] == NULL) {
 372                         printf("No memory\n");
 373                         return NULL;
 374                 }
 375         }
 376         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
 377         if (keys == NULL) {
 378                 printf("No memory\n");
 379                 return NULL;
 380         }
 381
 382         for (i = 0; i < TOTAL_ENTRY; i++)
 383                 keys[i] = i;
 384
 385         for (i = 0; i < TOTAL_ENTRY; i++) {
 386                 if (rte_hash_add_key_data(hash, keys + i,
 387                                 (void *)((uintptr_t)hash_data[i])) < 0) {
 388                         printf("Hash key add Failed #%d\n", i);
 389                         return NULL;
 390                 }
 391         }
 392         return hash;
 393 }
 394
 395 /*
 396  * Functional test:
 397  * Single writer, Single QS variable Single QSBR query, Blocking rcu_qsbr_check
 398  */
 399 static int
 400 test_rcu_qsbr_sw_sv_1qs(void)
 401 {
 402         uint64_t token, begin, cycles;
 403         int i, j, tmp_num_cores, sz;
 404         int32_t pos;
 405
 406         writer_done = 0;
 407
 408         rte_atomic64_clear(&updates);
 409         rte_atomic64_clear(&update_cycles);
 410         rte_atomic64_clear(&checks);
 411         rte_atomic64_clear(&check_cycles);
 412
 413         __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 414
 415         printf("\nPerf test: 1 writer, %d readers, 1 QSBR variable, 1 QSBR Query, Blocking QSBR Check\n", num_cores);
 416
 417         if (all_registered == 1)
 418                 tmp_num_cores = num_cores;
 419         else
 420                 tmp_num_cores = TEST_RCU_MAX_LCORE;
 421
 422         sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
 423         t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
 424                                                 RTE_CACHE_LINE_SIZE);
 425         /* QS variable is initialized */
 426         rte_rcu_qsbr_init(t[0], tmp_num_cores);
 427
 428         /* Shared data structure created */
 429         h = init_hash();
 430         if (h == NULL) {
 431                 printf("Hash init failed\n");
 432                 goto error;
 433         }
 434
 435         /* Reader threads are launched */
 436         for (i = 0; i < num_cores; i++)
 437                 rte_eal_remote_launch(test_rcu_qsbr_hash_reader, NULL,
 438                                         enabled_core_ids[i]);
 439
 440         begin = rte_rdtsc_precise();
 441
 442         for (i = 0; i < TOTAL_ENTRY; i++) {
 443                 /* Delete elements from the shared data structure */
 444                 pos = rte_hash_del_key(h, keys + i);
 445                 if (pos < 0) {
 446                         printf("Delete key failed #%d\n", keys[i]);
 447                         goto error;
 448                 }
 449                 /* Start the quiescent state query process */
 450                 token = rte_rcu_qsbr_start(t[0]);
 451
 452                 /* Check the quiescent state status */
 453                 rte_rcu_qsbr_check(t[0], token, true);
 454                 for (j = 0; j < tmp_num_cores; j++) {
 455                         if (hash_data[i][j] != COUNTER_VALUE &&
 456                                 hash_data[i][j] != 0) {
 457                                 printf("Reader thread ID %u did not complete #%d =  %d\n",
 458                                         j, i, hash_data[i][j]);
 459                                 goto error;
 460                         }
 461                 }
 462
 463                 if (rte_hash_free_key_with_position(h, pos) < 0) {
 464                         printf("Failed to free the key #%d\n", keys[i]);
 465                         goto error;
 466                 }
 467                 rte_free(hash_data[i]);
 468                 hash_data[i] = NULL;
 469         }
 470
 471         cycles = rte_rdtsc_precise() - begin;
 472         rte_atomic64_add(&check_cycles, cycles);
 473         rte_atomic64_add(&checks, i);
 474
 475         writer_done = 1;
 476
 477         /* Wait and check return value from reader threads */
 478         for (i = 0; i < num_cores; i++)
 479                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 480                         goto error;
 481         rte_hash_free(h);
 482         rte_free(keys);
 483
 484         printf("Following numbers include calls to rte_hash functions\n");
 485         printf("Cycles per 1 update(online/update/offline): %"PRIi64"\n",
 486                 rte_atomic64_read(&update_cycles) /
 487                 rte_atomic64_read(&updates));
 488
 489         printf("Cycles per 1 check(start, check): %"PRIi64"\n\n",
 490                 rte_atomic64_read(&check_cycles) /
 491                 rte_atomic64_read(&checks));
 492
 493         rte_free(t[0]);
 494
 495         return 0;
 496
 497 error:
 498         writer_done = 1;
 499         /* Wait until all readers have exited */
 500         rte_eal_mp_wait_lcore();
 501
 502         rte_hash_free(h);
 503         rte_free(keys);
 504         for (i = 0; i < TOTAL_ENTRY; i++)
 505                 rte_free(hash_data[i]);
 506
 507         rte_free(t[0]);
 508
 509         return -1;
 510 }
 511
 512 /*
 513  * Functional test:
 514  * Single writer, Single QS variable, Single QSBR query,
 515  * Non-blocking rcu_qsbr_check
 516  */
 517 static int
 518 test_rcu_qsbr_sw_sv_1qs_non_blocking(void)
 519 {
 520         uint64_t token, begin, cycles;
 521         int i, j, ret, tmp_num_cores, sz;
 522         int32_t pos;
 523
 524         writer_done = 0;
 525
 526         printf("Perf test: 1 writer, %d readers, 1 QSBR variable, 1 QSBR Query, Non-Blocking QSBR check\n", num_cores);
 527
 528         __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 529
 530         if (all_registered == 1)
 531                 tmp_num_cores = num_cores;
 532         else
 533                 tmp_num_cores = TEST_RCU_MAX_LCORE;
 534
 535         sz = rte_rcu_qsbr_get_memsize(tmp_num_cores);
 536         t[0] = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
 537                                                 RTE_CACHE_LINE_SIZE);
 538         /* QS variable is initialized */
 539         rte_rcu_qsbr_init(t[0], tmp_num_cores);
 540
 541         /* Shared data structure created */
 542         h = init_hash();
 543         if (h == NULL) {
 544                 printf("Hash init failed\n");
 545                 goto error;
 546         }
 547
 548         /* Reader threads are launched */
 549         for (i = 0; i < num_cores; i++)
 550                 rte_eal_remote_launch(test_rcu_qsbr_hash_reader, NULL,
 551                                         enabled_core_ids[i]);
 552
 553         begin = rte_rdtsc_precise();
 554
 555         for (i = 0; i < TOTAL_ENTRY; i++) {
 556                 /* Delete elements from the shared data structure */
 557                 pos = rte_hash_del_key(h, keys + i);
 558                 if (pos < 0) {
 559                         printf("Delete key failed #%d\n", keys[i]);
 560                         goto error;
 561                 }
 562                 /* Start the quiescent state query process */
 563                 token = rte_rcu_qsbr_start(t[0]);
 564
 565                 /* Check the quiescent state status */
 566                 do {
 567                         ret = rte_rcu_qsbr_check(t[0], token, false);
 568                 } while (ret == 0);
 569                 for (j = 0; j < tmp_num_cores; j++) {
 570                         if (hash_data[i][j] != COUNTER_VALUE &&
 571                                 hash_data[i][j] != 0) {
 572                                 printf("Reader thread ID %u did not complete #%d =  %d\n",
 573                                         j, i, hash_data[i][j]);
 574                                 goto error;
 575                         }
 576                 }
 577
 578                 if (rte_hash_free_key_with_position(h, pos) < 0) {
 579                         printf("Failed to free the key #%d\n", keys[i]);
 580                         goto error;
 581                 }
 582                 rte_free(hash_data[i]);
 583                 hash_data[i] = NULL;
 584         }
 585
 586         cycles = rte_rdtsc_precise() - begin;
 587         rte_atomic64_add(&check_cycles, cycles);
 588         rte_atomic64_add(&checks, i);
 589
 590         writer_done = 1;
 591         /* Wait and check return value from reader threads */
 592         for (i = 0; i < num_cores; i++)
 593                 if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 594                         goto error;
 595         rte_hash_free(h);
 596         rte_free(keys);
 597
 598         printf("Following numbers include calls to rte_hash functions\n");
 599         printf("Cycles per 1 update(online/update/offline): %"PRIi64"\n",
 600                 rte_atomic64_read(&update_cycles) /
 601                 rte_atomic64_read(&updates));
 602
 603         printf("Cycles per 1 check(start, check): %"PRIi64"\n\n",
 604                 rte_atomic64_read(&check_cycles) /
 605                 rte_atomic64_read(&checks));
 606
 607         rte_free(t[0]);
 608
 609         return 0;
 610
 611 error:
 612         writer_done = 1;
 613         /* Wait until all readers have exited */
 614         rte_eal_mp_wait_lcore();
 615
 616         rte_hash_free(h);
 617         rte_free(keys);
 618         for (i = 0; i < TOTAL_ENTRY; i++)
 619                 rte_free(hash_data[i]);
 620
 621         rte_free(t[0]);
 622
 623         return -1;
 624 }
 625
 626 static int
 627 test_rcu_qsbr_main(void)
 628 {
 629         rte_atomic64_init(&updates);
 630         rte_atomic64_init(&update_cycles);
 631         rte_atomic64_init(&checks);
 632         rte_atomic64_init(&check_cycles);
 633
 634         if (get_enabled_cores_mask() != 0)
 635                 return -1;
 636
 637         printf("Number of cores provided = %d\n", num_cores);
 638         if (num_cores < 2) {
 639                 printf("Test failed! Need 2 or more cores\n");
 640                 goto test_fail;
 641         }
 642         if (num_cores > TEST_RCU_MAX_LCORE) {
 643                 printf("Test failed! %d cores supported\n", TEST_RCU_MAX_LCORE);
 644                 goto test_fail;
 645         }
 646
 647         printf("Perf test with all reader threads registered\n");
 648         printf("--------------------------------------------\n");
 649         all_registered = 1;
 650
 651         if (test_rcu_qsbr_perf() < 0)
 652                 goto test_fail;
 653
 654         if (test_rcu_qsbr_rperf() < 0)
 655                 goto test_fail;
 656
 657         if (test_rcu_qsbr_wperf() < 0)
 658                 goto test_fail;
 659
 660         if (test_rcu_qsbr_sw_sv_1qs() < 0)
 661                 goto test_fail;
 662
 663         if (test_rcu_qsbr_sw_sv_1qs_non_blocking() < 0)
 664                 goto test_fail;
 665
 666         /* Make sure the actual number of cores provided is less than
 667          * TEST_RCU_MAX_LCORE. This will allow for some threads not
 668          * to be registered on the QS variable.
 669          */
 670         if (num_cores >= TEST_RCU_MAX_LCORE) {
 671                 printf("Test failed! number of cores provided should be less than %d\n",
 672                         TEST_RCU_MAX_LCORE);
 673                 goto test_fail;
 674         }
 675
 676         printf("Perf test with some of reader threads registered\n");
 677         printf("------------------------------------------------\n");
 678         all_registered = 0;
 679
 680         if (test_rcu_qsbr_perf() < 0)
 681                 goto test_fail;
 682
 683         if (test_rcu_qsbr_rperf() < 0)
 684                 goto test_fail;
 685
 686         if (test_rcu_qsbr_wperf() < 0)
 687                 goto test_fail;
 688
 689         if (test_rcu_qsbr_sw_sv_1qs() < 0)
 690                 goto test_fail;
 691
 692         if (test_rcu_qsbr_sw_sv_1qs_non_blocking() < 0)
 693                 goto test_fail;
 694
 695         printf("\n");
 696
 697         return 0;
 698
 699 test_fail:
 700         return -1;
 701 }
 702
 703 REGISTER_TEST_COMMAND(rcu_qsbr_perf_autotest, test_rcu_qsbr_main);