app/test/test_rwlock.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdint.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <sys/queue.h>
  10 #include <string.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_memory.h>
  14 #include <rte_per_lcore.h>
  15 #include <rte_launch.h>
  16 #include <rte_atomic.h>
  17 #include <rte_rwlock.h>
  18 #include <rte_eal.h>
  19 #include <rte_lcore.h>
  20 #include <rte_cycles.h>
  21
  22 #include "test.h"
  23
  24 /*
  25  * rwlock test
  26  * ===========
  27  * Provides UT for rte_rwlock API.
  28  * Main concern is on functional testing, but also provides some
  29  * performance measurements.
  30  * Obviously for proper testing need to be executed with more than one lcore.
  31  */
  32
  33 #define ITER_NUM        0x80
  34
  35 #define TEST_SEC        5
  36
  37 static rte_rwlock_t sl;
  38 static rte_rwlock_t sl_tab[RTE_MAX_LCORE];
  39 static rte_atomic32_t synchro;
  40
  41 enum {
  42         LC_TYPE_RDLOCK,
  43         LC_TYPE_WRLOCK,
  44 };
  45
  46 static struct {
  47         rte_rwlock_t lock;
  48         uint64_t tick;
  49         volatile union {
  50                 uint8_t u8[RTE_CACHE_LINE_SIZE];
  51                 uint64_t u64[RTE_CACHE_LINE_SIZE / sizeof(uint64_t)];
  52         } data;
  53 } __rte_cache_aligned try_rwlock_data;
  54
  55 struct try_rwlock_lcore {
  56         int32_t rc;
  57         int32_t type;
  58         struct {
  59                 uint64_t tick;
  60                 uint64_t fail;
  61                 uint64_t success;
  62         } stat;
  63 } __rte_cache_aligned;
  64
  65 static struct try_rwlock_lcore try_lcore_data[RTE_MAX_LCORE];
  66
  67 static int
  68 test_rwlock_per_core(__rte_unused void *arg)
  69 {
  70         rte_rwlock_write_lock(&sl);
  71         printf("Global write lock taken on core %u\n", rte_lcore_id());
  72         rte_rwlock_write_unlock(&sl);
  73
  74         rte_rwlock_write_lock(&sl_tab[rte_lcore_id()]);
  75         printf("Hello from core %u !\n", rte_lcore_id());
  76         rte_rwlock_write_unlock(&sl_tab[rte_lcore_id()]);
  77
  78         rte_rwlock_read_lock(&sl);
  79         printf("Global read lock taken on core %u\n", rte_lcore_id());
  80         rte_delay_ms(100);
  81         printf("Release global read lock on core %u\n", rte_lcore_id());
  82         rte_rwlock_read_unlock(&sl);
  83
  84         return 0;
  85 }
  86
  87 static rte_rwlock_t lk = RTE_RWLOCK_INITIALIZER;
  88 static volatile uint64_t rwlock_data;
  89 static uint64_t time_count[RTE_MAX_LCORE] = {0};
  90
  91 #define MAX_LOOP 10000
  92 #define TEST_RWLOCK_DEBUG 0
  93
  94 static int
  95 load_loop_fn(__rte_unused void *arg)
  96 {
  97         uint64_t time_diff = 0, begin;
  98         uint64_t hz = rte_get_timer_hz();
  99         uint64_t lcount = 0;
 100         const unsigned int lcore = rte_lcore_id();
 101
 102         /* wait synchro for slaves */
 103         if (lcore != rte_get_master_lcore())
 104                 while (rte_atomic32_read(&synchro) == 0)
 105                         ;
 106
 107         begin = rte_rdtsc_precise();
 108         while (lcount < MAX_LOOP) {
 109                 rte_rwlock_write_lock(&lk);
 110                 ++rwlock_data;
 111                 rte_rwlock_write_unlock(&lk);
 112
 113                 rte_rwlock_read_lock(&lk);
 114                 if (TEST_RWLOCK_DEBUG && !(lcount % 100))
 115                         printf("Core [%u] rwlock_data = %"PRIu64"\n",
 116                                 lcore, rwlock_data);
 117                 rte_rwlock_read_unlock(&lk);
 118
 119                 lcount++;
 120                 /* delay to make lock duty cycle slightly realistic */
 121                 rte_pause();
 122         }
 123
 124         time_diff = rte_rdtsc_precise() - begin;
 125         time_count[lcore] = time_diff * 1000000 / hz;
 126         return 0;
 127 }
 128
 129 static int
 130 test_rwlock_perf(void)
 131 {
 132         unsigned int i;
 133         uint64_t total = 0;
 134
 135         printf("\nRwlock Perf Test on %u cores...\n", rte_lcore_count());
 136
 137         /* clear synchro and start slaves */
 138         rte_atomic32_set(&synchro, 0);
 139         if (rte_eal_mp_remote_launch(load_loop_fn, NULL, SKIP_MASTER) < 0)
 140                 return -1;
 141
 142         /* start synchro and launch test on master */
 143         rte_atomic32_set(&synchro, 1);
 144         load_loop_fn(NULL);
 145
 146         rte_eal_mp_wait_lcore();
 147
 148         RTE_LCORE_FOREACH(i) {
 149                 printf("Core [%u] cost time = %"PRIu64" us\n",
 150                         i, time_count[i]);
 151                 total += time_count[i];
 152         }
 153
 154         printf("Total cost time = %"PRIu64" us\n", total);
 155         memset(time_count, 0, sizeof(time_count));
 156
 157         return 0;
 158 }
 159
 160 /*
 161  * - There is a global rwlock and a table of rwlocks (one per lcore).
 162  *
 163  * - The test function takes all of these locks and launches the
 164  *   ``test_rwlock_per_core()`` function on each core (except the master).
 165  *
 166  *   - The function takes the global write lock, display something,
 167  *     then releases the global lock.
 168  *   - Then, it takes the per-lcore write lock, display something, and
 169  *     releases the per-core lock.
 170  *   - Finally, a read lock is taken during 100 ms, then released.
 171  *
 172  * - The main function unlocks the per-lcore locks sequentially and
 173  *   waits between each lock. This triggers the display of a message
 174  *   for each core, in the correct order.
 175  *
 176  *   Then, it tries to take the global write lock and display the last
 177  *   message. The autotest script checks that the message order is correct.
 178  */
 179 static int
 180 rwlock_test1(void)
 181 {
 182         int i;
 183
 184         rte_rwlock_init(&sl);
 185         for (i=0; i<RTE_MAX_LCORE; i++)
 186                 rte_rwlock_init(&sl_tab[i]);
 187
 188         rte_rwlock_write_lock(&sl);
 189
 190         RTE_LCORE_FOREACH_SLAVE(i) {
 191                 rte_rwlock_write_lock(&sl_tab[i]);
 192                 rte_eal_remote_launch(test_rwlock_per_core, NULL, i);
 193         }
 194
 195         rte_rwlock_write_unlock(&sl);
 196
 197         RTE_LCORE_FOREACH_SLAVE(i) {
 198                 rte_rwlock_write_unlock(&sl_tab[i]);
 199                 rte_delay_ms(100);
 200         }
 201
 202         rte_rwlock_write_lock(&sl);
 203         /* this message should be the last message of test */
 204         printf("Global write lock taken on master core %u\n", rte_lcore_id());
 205         rte_rwlock_write_unlock(&sl);
 206
 207         rte_eal_mp_wait_lcore();
 208
 209         if (test_rwlock_perf() < 0)
 210                 return -1;
 211
 212         return 0;
 213 }
 214
 215 static int
 216 try_read(uint32_t lc)
 217 {
 218         int32_t rc;
 219         uint32_t i;
 220
 221         rc = rte_rwlock_read_trylock(&try_rwlock_data.lock);
 222         if (rc != 0)
 223                 return rc;
 224
 225         for (i = 0; i != RTE_DIM(try_rwlock_data.data.u64); i++) {
 226
 227                 /* race condition occurred, lock doesn't work properly */
 228                 if (try_rwlock_data.data.u64[i] != 0) {
 229                         printf("%s(%u) error: unexpected data pattern\n",
 230                                 __func__, lc);
 231                         rte_memdump(stdout, NULL,
 232                                 (void *)(uintptr_t)&try_rwlock_data.data,
 233                                 sizeof(try_rwlock_data.data));
 234                         rc = -EFAULT;
 235                         break;
 236                 }
 237         }
 238
 239         rte_rwlock_read_unlock(&try_rwlock_data.lock);
 240         return rc;
 241 }
 242
 243 static int
 244 try_write(uint32_t lc)
 245 {
 246         int32_t rc;
 247         uint32_t i, v;
 248
 249         v = RTE_MAX(lc % UINT8_MAX, 1U);
 250
 251         rc = rte_rwlock_write_trylock(&try_rwlock_data.lock);
 252         if (rc != 0)
 253                 return rc;
 254
 255         /* update by bytes in reverese order */
 256         for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) {
 257
 258                 /* race condition occurred, lock doesn't work properly */
 259                 if (try_rwlock_data.data.u8[i] != 0) {
 260                         printf("%s:%d(%u) error: unexpected data pattern\n",
 261                                 __func__, __LINE__, lc);
 262                         rte_memdump(stdout, NULL,
 263                                 (void *)(uintptr_t)&try_rwlock_data.data,
 264                                 sizeof(try_rwlock_data.data));
 265                         rc = -EFAULT;
 266                         break;
 267                 }
 268
 269                 try_rwlock_data.data.u8[i] = v;
 270         }
 271
 272         /* restore by bytes in reverese order */
 273         for (i = RTE_DIM(try_rwlock_data.data.u8); i-- != 0; ) {
 274
 275                 /* race condition occurred, lock doesn't work properly */
 276                 if (try_rwlock_data.data.u8[i] != v) {
 277                         printf("%s:%d(%u) error: unexpected data pattern\n",
 278                                 __func__, __LINE__, lc);
 279                         rte_memdump(stdout, NULL,
 280                                 (void *)(uintptr_t)&try_rwlock_data.data,
 281                                 sizeof(try_rwlock_data.data));
 282                         rc = -EFAULT;
 283                         break;
 284                 }
 285
 286                 try_rwlock_data.data.u8[i] = 0;
 287         }
 288
 289         rte_rwlock_write_unlock(&try_rwlock_data.lock);
 290         return rc;
 291 }
 292
 293 static int
 294 try_read_lcore(__rte_unused void *data)
 295 {
 296         int32_t rc;
 297         uint32_t i, lc;
 298         uint64_t ftm, stm, tm;
 299         struct try_rwlock_lcore *lcd;
 300
 301         lc = rte_lcore_id();
 302         lcd = try_lcore_data + lc;
 303         lcd->type = LC_TYPE_RDLOCK;
 304
 305         ftm = try_rwlock_data.tick;
 306         stm = rte_get_timer_cycles();
 307
 308         do {
 309                 for (i = 0; i != ITER_NUM; i++) {
 310                         rc = try_read(lc);
 311                         if (rc == 0)
 312                                 lcd->stat.success++;
 313                         else if (rc == -EBUSY)
 314                                 lcd->stat.fail++;
 315                         else
 316                                 break;
 317                         rc = 0;
 318                 }
 319                 tm = rte_get_timer_cycles() - stm;
 320         } while (tm < ftm && rc == 0);
 321
 322         lcd->rc = rc;
 323         lcd->stat.tick = tm;
 324         return rc;
 325 }
 326
 327 static int
 328 try_write_lcore(__rte_unused void *data)
 329 {
 330         int32_t rc;
 331         uint32_t i, lc;
 332         uint64_t ftm, stm, tm;
 333         struct try_rwlock_lcore *lcd;
 334
 335         lc = rte_lcore_id();
 336         lcd = try_lcore_data + lc;
 337         lcd->type = LC_TYPE_WRLOCK;
 338
 339         ftm = try_rwlock_data.tick;
 340         stm = rte_get_timer_cycles();
 341
 342         do {
 343                 for (i = 0; i != ITER_NUM; i++) {
 344                         rc = try_write(lc);
 345                         if (rc == 0)
 346                                 lcd->stat.success++;
 347                         else if (rc == -EBUSY)
 348                                 lcd->stat.fail++;
 349                         else
 350                                 break;
 351                         rc = 0;
 352                 }
 353                 tm = rte_get_timer_cycles() - stm;
 354         } while (tm < ftm && rc == 0);
 355
 356         lcd->rc = rc;
 357         lcd->stat.tick = tm;
 358         return rc;
 359 }
 360
 361 static void
 362 print_try_lcore_stats(const struct try_rwlock_lcore *tlc, uint32_t lc)
 363 {
 364         uint64_t f, s;
 365
 366         f = RTE_MAX(tlc->stat.fail, 1ULL);
 367         s = RTE_MAX(tlc->stat.success, 1ULL);
 368
 369         printf("try_lcore_data[%u]={\n"
 370                 "\trc=%d,\n"
 371                 "\ttype=%s,\n"
 372                 "\tfail=%" PRIu64 ",\n"
 373                 "\tsuccess=%" PRIu64 ",\n"
 374                 "\tcycles=%" PRIu64 ",\n"
 375                 "\tcycles/op=%#Lf,\n"
 376                 "\tcycles/success=%#Lf,\n"
 377                 "\tsuccess/fail=%#Lf,\n"
 378                 "};\n",
 379                 lc,
 380                 tlc->rc,
 381                 tlc->type == LC_TYPE_RDLOCK ? "RDLOCK" : "WRLOCK",
 382                 tlc->stat.fail,
 383                 tlc->stat.success,
 384                 tlc->stat.tick,
 385                 (long double)tlc->stat.tick /
 386                 (tlc->stat.fail + tlc->stat.success),
 387                 (long double)tlc->stat.tick / s,
 388                 (long double)tlc->stat.success / f);
 389 }
 390
 391 static void
 392 collect_try_lcore_stats(struct try_rwlock_lcore *tlc,
 393         const struct try_rwlock_lcore *lc)
 394 {
 395         tlc->stat.tick += lc->stat.tick;
 396         tlc->stat.fail += lc->stat.fail;
 397         tlc->stat.success += lc->stat.success;
 398 }
 399
 400 /*
 401  * Process collected results:
 402  *  - check status
 403  *  - collect and print statistics
 404  */
 405 static int
 406 process_try_lcore_stats(void)
 407 {
 408         int32_t rc;
 409         uint32_t lc, rd, wr;
 410         struct try_rwlock_lcore rlc, wlc;
 411
 412         memset(&rlc, 0, sizeof(rlc));
 413         memset(&wlc, 0, sizeof(wlc));
 414
 415         rlc.type = LC_TYPE_RDLOCK;
 416         wlc.type = LC_TYPE_WRLOCK;
 417         rd = 0;
 418         wr = 0;
 419
 420         rc = 0;
 421         RTE_LCORE_FOREACH(lc) {
 422                 rc |= try_lcore_data[lc].rc;
 423                 if (try_lcore_data[lc].type == LC_TYPE_RDLOCK) {
 424                         collect_try_lcore_stats(&rlc, try_lcore_data + lc);
 425                         rd++;
 426                 } else {
 427                         collect_try_lcore_stats(&wlc, try_lcore_data + lc);
 428                         wr++;
 429                 }
 430         }
 431
 432         if (rc == 0) {
 433                 RTE_LCORE_FOREACH(lc)
 434                         print_try_lcore_stats(try_lcore_data + lc, lc);
 435
 436                 if (rd != 0) {
 437                         printf("aggregated stats for %u RDLOCK cores:\n", rd);
 438                         print_try_lcore_stats(&rlc, rd);
 439                 }
 440
 441                 if (wr != 0) {
 442                         printf("aggregated stats for %u WRLOCK cores:\n", wr);
 443                         print_try_lcore_stats(&wlc, wr);
 444                 }
 445         }
 446
 447         return rc;
 448 }
 449
 450 static void
 451 try_test_reset(void)
 452 {
 453         memset(&try_lcore_data, 0, sizeof(try_lcore_data));
 454         memset(&try_rwlock_data, 0, sizeof(try_rwlock_data));
 455         try_rwlock_data.tick = TEST_SEC * rte_get_tsc_hz();
 456 }
 457
 458 /* all lcores grab RDLOCK */
 459 static int
 460 try_rwlock_test_rda(void)
 461 {
 462         try_test_reset();
 463
 464         /* start read test on all avaialble lcores */
 465         rte_eal_mp_remote_launch(try_read_lcore, NULL, CALL_MASTER);
 466         rte_eal_mp_wait_lcore();
 467
 468         return process_try_lcore_stats();
 469 }
 470
 471 /* all slave lcores grab RDLOCK, master one grabs WRLOCK */
 472 static int
 473 try_rwlock_test_rds_wrm(void)
 474 {
 475         try_test_reset();
 476
 477         rte_eal_mp_remote_launch(try_read_lcore, NULL, SKIP_MASTER);
 478         try_write_lcore(NULL);
 479         rte_eal_mp_wait_lcore();
 480
 481         return process_try_lcore_stats();
 482 }
 483
 484 /* master and even slave lcores grab RDLOCK, odd lcores grab WRLOCK */
 485 static int
 486 try_rwlock_test_rde_wro(void)
 487 {
 488         uint32_t lc, mlc;
 489
 490         try_test_reset();
 491
 492         mlc = rte_get_master_lcore();
 493
 494         RTE_LCORE_FOREACH(lc) {
 495                 if (lc != mlc) {
 496                         if ((lc & 1) == 0)
 497                                 rte_eal_remote_launch(try_read_lcore,
 498                                                 NULL, lc);
 499                         else
 500                                 rte_eal_remote_launch(try_write_lcore,
 501                                                 NULL, lc);
 502                 }
 503         }
 504         try_read_lcore(NULL);
 505         rte_eal_mp_wait_lcore();
 506
 507         return process_try_lcore_stats();
 508 }
 509
 510 static int
 511 test_rwlock(void)
 512 {
 513         uint32_t i;
 514         int32_t rc, ret;
 515
 516         static const struct {
 517                 const char *name;
 518                 int (*ftst)(void);
 519         } test[] = {
 520                 {
 521                         .name = "rwlock_test1",
 522                         .ftst = rwlock_test1,
 523                 },
 524                 {
 525                         .name = "try_rwlock_test_rda",
 526                         .ftst = try_rwlock_test_rda,
 527                 },
 528                 {
 529                         .name = "try_rwlock_test_rds_wrm",
 530                         .ftst = try_rwlock_test_rds_wrm,
 531                 },
 532                 {
 533                         .name = "try_rwlock_test_rde_wro",
 534                         .ftst = try_rwlock_test_rde_wro,
 535                 },
 536         };
 537
 538         ret = 0;
 539         for (i = 0; i != RTE_DIM(test); i++) {
 540                 printf("starting test %s;\n", test[i].name);
 541                 rc = test[i].ftst();
 542                 printf("test %s completed with status %d\n", test[i].name, rc);
 543                 ret |= rc;
 544         }
 545
 546         return ret;
 547 }
 548
 549 REGISTER_TEST_COMMAND(rwlock_autotest, test_rwlock);
 550
 551 /* subtests used in meson for CI */
 552 REGISTER_TEST_COMMAND(rwlock_test1_autotest, rwlock_test1);
 553 REGISTER_TEST_COMMAND(rwlock_rda_autotest, try_rwlock_test_rda);
 554 REGISTER_TEST_COMMAND(rwlock_rds_wrm_autotest, try_rwlock_test_rds_wrm);
 555 REGISTER_TEST_COMMAND(rwlock_rde_wro_autotest, try_rwlock_test_rde_wro);