tests/regression/rcutorture.h

   1 // SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation.
   2 //
   3 // SPDX-License-Identifier: GPL-2.0-or-later
   4
   5 /*
   6  * rcutorture.h: simple user-level performance/stress test of RCU.
   7  *
   8  * Usage:
   9  *      ./rcu <nreaders> rperf [ <cpustride> ]
  10  *              Run a read-side performance test with the specified
  11  *              number of readers spaced by <cpustride>.
  12  *              Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
  13  *              CPUs from 0 to 30.
  14  *      ./rcu <nupdaters> uperf [ <cpustride> ]
  15  *              Run an update-side performance test with the specified
  16  *              number of updaters and specified CPU spacing.
  17  *      ./rcu <nreaders> perf [ <cpustride> ]
  18  *              Run a combined read/update performance test with the specified
  19  *              number of readers and one updater and specified CPU spacing.
  20  *              The readers run on the low-numbered CPUs and the updater
  21  *              of the highest-numbered CPU.
  22  *
  23  * The above tests produce output as follows:
  24  *
  25  * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
  26  * ns/read: 43.4707  ns/update: 6848.1
  27  *
  28  * The first line lists the total number of RCU reads and updates executed
  29  * during the test, the number of reader threads, the number of updater
  30  * threads, and the duration of the test in seconds.  The second line
  31  * lists the average duration of each type of operation in nanoseconds,
  32  * or "nan" if the corresponding type of operation was not performed.
  33  *
  34  *      ./rcu <nreaders> stress
  35  *              Run a stress test with the specified number of readers and
  36  *              one updater.  None of the threads are affinitied to any
  37  *              particular CPU.
  38  *
  39  * This test produces output as follows:
  40  *
  41  * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
  42  * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
  43  *
  44  * The first line lists the number of RCU read and update operations
  45  * executed, followed by the number of memory-ordering violations
  46  * (which will be zero in a correct RCU implementation).  The second
  47  * line lists the number of readers observing progressively more stale
  48  * data.  A correct RCU implementation will have all but the first two
  49  * numbers non-zero.
  50  */
  51
  52 /*
  53  * Test variables.
  54  */
  55
  56 #include <stdlib.h>
  57 #include "tap.h"
  58
  59 #define NR_TESTS        1
  60
  61 DEFINE_PER_THREAD(long long, n_reads_pt);
  62 DEFINE_PER_THREAD(long long, n_updates_pt);
  63
  64 enum callrcu_type {
  65         CALLRCU_GLOBAL,
  66         CALLRCU_PERCPU,
  67         CALLRCU_PERTHREAD,
  68 };
  69
  70 enum writer_state {
  71         WRITER_STATE_SYNC_RCU,
  72         WRITER_STATE_CALL_RCU,
  73         WRITER_STATE_POLL_RCU,
  74 };
  75
  76 static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
  77
  78 long long n_reads = 0LL;
  79 long n_updates = 0L;
  80 int nthreadsrunning;
  81 char argsbuf[64];
  82
  83 #define GOFLAG_INIT 0
  84 #define GOFLAG_RUN  1
  85 #define GOFLAG_STOP 2
  86
  87 volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
  88         = GOFLAG_INIT;
  89
  90 #define RCU_READ_RUN 1000
  91
  92 //MD
  93 #define RCU_READ_NESTABLE
  94
  95 #ifdef RCU_READ_NESTABLE
  96 #define rcu_read_lock_nest() rcu_read_lock()
  97 #define rcu_read_unlock_nest() rcu_read_unlock()
  98 #else /* #ifdef RCU_READ_NESTABLE */
  99 #define rcu_read_lock_nest()
 100 #define rcu_read_unlock_nest()
 101 #endif /* #else #ifdef RCU_READ_NESTABLE */
 102
 103 #ifdef TORTURE_QSBR
 104 #define mark_rcu_quiescent_state        rcu_quiescent_state
 105 #define put_thread_offline              rcu_thread_offline
 106 #define put_thread_online               rcu_thread_online
 107 #endif
 108
 109 #ifndef mark_rcu_quiescent_state
 110 #define mark_rcu_quiescent_state() do {} while (0)
 111 #endif /* #ifdef mark_rcu_quiescent_state */
 112
 113 #ifndef put_thread_offline
 114 #define put_thread_offline()            do {} while (0)
 115 #define put_thread_online()             do {} while (0)
 116 #define put_thread_online_delay()       do {} while (0)
 117 #else /* #ifndef put_thread_offline */
 118 #define put_thread_online_delay()       synchronize_rcu()
 119 #endif /* #else #ifndef put_thread_offline */
 120
 121 /*
 122  * Performance test.
 123  */
 124
 125 static
 126 void *rcu_read_perf_test(void *arg)
 127 {
 128         int i;
 129         int me = (long)arg;
 130         long long n_reads_local = 0;
 131
 132         rcu_register_thread();
 133         run_on(me);
 134         uatomic_inc(&nthreadsrunning);
 135         put_thread_offline();
 136         while (goflag == GOFLAG_INIT)
 137                 (void) poll(NULL, 0, 1);
 138         put_thread_online();
 139         while (goflag == GOFLAG_RUN) {
 140                 for (i = 0; i < RCU_READ_RUN; i++) {
 141                         rcu_read_lock();
 142                         /* rcu_read_lock_nest(); */
 143                         /* rcu_read_unlock_nest(); */
 144                         rcu_read_unlock();
 145                 }
 146                 n_reads_local += RCU_READ_RUN;
 147                 mark_rcu_quiescent_state();
 148         }
 149         __get_thread_var(n_reads_pt) += n_reads_local;
 150         put_thread_offline();
 151         rcu_unregister_thread();
 152
 153         return (NULL);
 154 }
 155
 156 static
 157 void *rcu_update_perf_test(void *arg __attribute__((unused)))
 158 {
 159         long long n_updates_local = 0;
 160
 161         if (callrcu_type == CALLRCU_PERTHREAD) {
 162                 struct call_rcu_data *crdp;
 163
 164                 crdp = create_call_rcu_data(0, -1);
 165                 if (crdp != NULL) {
 166                         diag("Successfully using per-thread call_rcu() worker.");
 167                         set_thread_call_rcu_data(crdp);
 168                 }
 169         }
 170         uatomic_inc(&nthreadsrunning);
 171         while (goflag == GOFLAG_INIT)
 172                 (void) poll(NULL, 0, 1);
 173         while (goflag == GOFLAG_RUN) {
 174                 synchronize_rcu();
 175                 n_updates_local++;
 176         }
 177         __get_thread_var(n_updates_pt) += n_updates_local;
 178         if (callrcu_type == CALLRCU_PERTHREAD) {
 179                 struct call_rcu_data *crdp;
 180
 181                 crdp = get_thread_call_rcu_data();
 182                 set_thread_call_rcu_data(NULL);
 183                 call_rcu_data_free(crdp);
 184         }
 185         return NULL;
 186 }
 187
 188 static
 189 void perftestinit(void)
 190 {
 191         init_per_thread(n_reads_pt, 0LL);
 192         init_per_thread(n_updates_pt, 0LL);
 193         uatomic_set(&nthreadsrunning, 0);
 194 }
 195
 196 static
 197 int perftestrun(int nthreads, int nreaders, int nupdaters)
 198 {
 199         int t;
 200         int duration = 1;
 201
 202         cmm_smp_mb();
 203         while (uatomic_read(&nthreadsrunning) < nthreads)
 204                 (void) poll(NULL, 0, 1);
 205         goflag = GOFLAG_RUN;
 206         cmm_smp_mb();
 207         sleep(duration);
 208         cmm_smp_mb();
 209         goflag = GOFLAG_STOP;
 210         cmm_smp_mb();
 211         wait_all_threads();
 212         for_each_thread(t) {
 213                 n_reads += per_thread(n_reads_pt, t);
 214                 n_updates += per_thread(n_updates_pt, t);
 215         }
 216         diag("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d",
 217                n_reads, n_updates, nreaders, nupdaters, duration);
 218         diag("ns/read: %g  ns/update: %g",
 219                ((duration * 1000*1000*1000.*(double)nreaders) /
 220                 (double)n_reads),
 221                ((duration * 1000*1000*1000.*(double)nupdaters) /
 222                 (double)n_updates));
 223         if (get_cpu_call_rcu_data(0)) {
 224                 diag("Deallocating per-CPU call_rcu threads.\n");
 225                 free_all_cpu_call_rcu_data();
 226         }
 227         return 0;
 228 }
 229
 230 static
 231 int perftest(int nreaders, int cpustride)
 232 {
 233         int i;
 234         long arg;
 235
 236         perftestinit();
 237         for (i = 0; i < nreaders; i++) {
 238                 arg = (long)(i * cpustride);
 239                 create_thread(rcu_read_perf_test, (void *)arg);
 240         }
 241         arg = (long)(i * cpustride);
 242         create_thread(rcu_update_perf_test, (void *)arg);
 243         return perftestrun(i + 1, nreaders, 1);
 244 }
 245
 246 static
 247 int rperftest(int nreaders, int cpustride)
 248 {
 249         int i;
 250         long arg;
 251
 252         perftestinit();
 253         init_per_thread(n_reads_pt, 0LL);
 254         for (i = 0; i < nreaders; i++) {
 255                 arg = (long)(i * cpustride);
 256                 create_thread(rcu_read_perf_test, (void *)arg);
 257         }
 258         return perftestrun(i, nreaders, 0);
 259 }
 260
 261 static
 262 int uperftest(int nupdaters, int cpustride)
 263 {
 264         int i;
 265         long arg;
 266
 267         perftestinit();
 268         init_per_thread(n_reads_pt, 0LL);
 269         for (i = 0; i < nupdaters; i++) {
 270                 arg = (long)(i * cpustride);
 271                 create_thread(rcu_update_perf_test, (void *)arg);
 272         }
 273         return perftestrun(i, 0, nupdaters);
 274 }
 275
 276 /*
 277  * Stress test.
 278  */
 279
 280 #define RCU_STRESS_PIPE_LEN 10
 281
 282 struct rcu_stress {
 283         int pipe_count;
 284         int mbtest;
 285 };
 286
 287 struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0, 0 } };
 288 struct rcu_stress *rcu_stress_current;
 289 int rcu_stress_idx = 0;
 290
 291 int n_mberror = 0;
 292 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
 293
 294 int garbage = 0;
 295
 296 static
 297 void *rcu_read_stress_test(void *arg __attribute__((unused)))
 298 {
 299         int i;
 300         int itercnt = 0;
 301         struct rcu_stress *p;
 302         int pc;
 303
 304         rcu_register_thread();
 305         put_thread_offline();
 306         while (goflag == GOFLAG_INIT)
 307                 (void) poll(NULL, 0, 1);
 308         put_thread_online();
 309         while (goflag == GOFLAG_RUN) {
 310                 rcu_read_lock();
 311                 p = rcu_dereference(rcu_stress_current);
 312                 if (p->mbtest == 0)
 313                         n_mberror++;
 314                 rcu_read_lock_nest();
 315                 for (i = 0; i < 100; i++)
 316                         garbage++;
 317                 rcu_read_unlock_nest();
 318                 pc = p->pipe_count;
 319                 rcu_read_unlock();
 320                 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
 321                         pc = RCU_STRESS_PIPE_LEN;
 322                 __get_thread_var(rcu_stress_count)[pc]++;
 323                 __get_thread_var(n_reads_pt)++;
 324                 mark_rcu_quiescent_state();
 325                 if ((++itercnt % 0x1000) == 0) {
 326                         put_thread_offline();
 327                         put_thread_online_delay();
 328                         put_thread_online();
 329                 }
 330         }
 331         put_thread_offline();
 332         rcu_unregister_thread();
 333
 334         return (NULL);
 335 }
 336
 337 static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
 338 static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
 339 static bool call_rcu_wait;
 340
 341 static
 342 void rcu_update_stress_test_rcu(struct rcu_head *head __attribute__((unused)))
 343 {
 344         int ret;
 345
 346         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 347         if (ret) {
 348                 errno = ret;
 349                 diag("pthread_mutex_lock: %s",
 350                         strerror(errno));
 351                 abort();
 352         }
 353         ret = pthread_cond_signal(&call_rcu_test_cond);
 354         if (ret) {
 355                 errno = ret;
 356                 diag("pthread_cond_signal: %s",
 357                         strerror(errno));
 358                 abort();
 359         }
 360         call_rcu_wait = false;
 361         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 362         if (ret) {
 363                 errno = ret;
 364                 diag("pthread_mutex_unlock: %s",
 365                         strerror(errno));
 366                 abort();
 367         }
 368 }
 369
 370 static
 371 void advance_writer_state(enum writer_state *state)
 372 {
 373         switch (*state) {
 374         case WRITER_STATE_SYNC_RCU:
 375                 *state = WRITER_STATE_CALL_RCU;
 376                 break;
 377         case WRITER_STATE_CALL_RCU:
 378                 *state = WRITER_STATE_POLL_RCU;
 379                 break;
 380         case WRITER_STATE_POLL_RCU:
 381                 *state = WRITER_STATE_SYNC_RCU;
 382                 break;
 383         }
 384 }
 385
 386 static
 387 void *rcu_update_stress_test(void *arg __attribute__((unused)))
 388 {
 389         int i;
 390         struct rcu_stress *p;
 391         struct rcu_head rh;
 392         enum writer_state writer_state = WRITER_STATE_SYNC_RCU;
 393
 394         while (goflag == GOFLAG_INIT)
 395                 (void) poll(NULL, 0, 1);
 396         while (goflag == GOFLAG_RUN) {
 397                 i = rcu_stress_idx + 1;
 398                 if (i >= RCU_STRESS_PIPE_LEN)
 399                         i = 0;
 400                 p = &rcu_stress_array[i];
 401                 p->mbtest = 0;
 402                 cmm_smp_mb();
 403                 p->pipe_count = 0;
 404                 p->mbtest = 1;
 405                 rcu_assign_pointer(rcu_stress_current, p);
 406                 rcu_stress_idx = i;
 407                 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
 408                         if (i != rcu_stress_idx)
 409                                 rcu_stress_array[i].pipe_count++;
 410                 switch (writer_state) {
 411                 case WRITER_STATE_SYNC_RCU:
 412                         synchronize_rcu();
 413                         break;
 414                 case WRITER_STATE_CALL_RCU:
 415                 {
 416                         int ret;
 417
 418                         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 419                         if (ret) {
 420                                 errno = ret;
 421                                 diag("pthread_mutex_lock: %s",
 422                                         strerror(errno));
 423                                 abort();
 424                         }
 425                         rcu_register_thread();
 426                         call_rcu(&rh, rcu_update_stress_test_rcu);
 427                         rcu_unregister_thread();
 428                         /*
 429                          * Our MacOS X test machine with the following
 430                          * config:
 431                          * 15.6.0 Darwin Kernel Version 15.6.0
 432                          * root:xnu-3248.60.10~1/RELEASE_X86_64
 433                          * appears to have issues with liburcu-signal
 434                          * signal being delivered on top of
 435                          * pthread_cond_wait. It seems to make the
 436                          * thread continue, and therefore corrupt the
 437                          * rcu_head. Work around this issue by
 438                          * unregistering the RCU read-side thread
 439                          * immediately after call_rcu (call_rcu needs
 440                          * us to be registered RCU readers).
 441                          */
 442                         call_rcu_wait = true;
 443                         do {
 444                                 ret = pthread_cond_wait(&call_rcu_test_cond,
 445                                                         &call_rcu_test_mutex);
 446                         } while (call_rcu_wait);
 447                         if (ret) {
 448                                 errno = ret;
 449                                 diag("pthread_cond_signal: %s",
 450                                         strerror(errno));
 451                                 abort();
 452                         }
 453                         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 454                         if (ret) {
 455                                 errno = ret;
 456                                 diag("pthread_mutex_unlock: %s",
 457                                         strerror(errno));
 458                                 abort();
 459                         }
 460                         break;
 461                 }
 462                 case WRITER_STATE_POLL_RCU:
 463                 {
 464                         struct urcu_gp_poll_state poll_state;
 465
 466                         rcu_register_thread();
 467                         poll_state = start_poll_synchronize_rcu();
 468                         rcu_unregister_thread();
 469                         while (!poll_state_synchronize_rcu(poll_state))
 470                                 (void) poll(NULL, 0, 1);        /* Wait for 1ms */
 471                         break;
 472                 }
 473                 }
 474                 n_updates++;
 475                 advance_writer_state(&writer_state);
 476         }
 477
 478         return NULL;
 479 }
 480
 481 static
 482 void *rcu_fake_update_stress_test(void *arg __attribute__((unused)))
 483 {
 484         if (callrcu_type == CALLRCU_PERTHREAD) {
 485                 struct call_rcu_data *crdp;
 486
 487                 crdp = create_call_rcu_data(0, -1);
 488                 if (crdp != NULL) {
 489                         diag("Successfully using per-thread call_rcu() worker.");
 490                         set_thread_call_rcu_data(crdp);
 491                 }
 492         }
 493         while (goflag == GOFLAG_INIT)
 494                 (void) poll(NULL, 0, 1);
 495         while (goflag == GOFLAG_RUN) {
 496                 synchronize_rcu();
 497                 (void) poll(NULL, 0, 1);
 498         }
 499         if (callrcu_type == CALLRCU_PERTHREAD) {
 500                 struct call_rcu_data *crdp;
 501
 502                 crdp = get_thread_call_rcu_data();
 503                 set_thread_call_rcu_data(NULL);
 504                 call_rcu_data_free(crdp);
 505         }
 506         return NULL;
 507 }
 508
 509 static
 510 int stresstest(int nreaders)
 511 {
 512         int i;
 513         int t;
 514         long long *p;
 515         long long sum;
 516
 517         init_per_thread(n_reads_pt, 0LL);
 518         for_each_thread(t) {
 519                 p = &per_thread(rcu_stress_count,t)[0];
 520                 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
 521                         p[i] = 0LL;
 522         }
 523         rcu_stress_current = &rcu_stress_array[0];
 524         rcu_stress_current->pipe_count = 0;
 525         rcu_stress_current->mbtest = 1;
 526         for (i = 0; i < nreaders; i++)
 527                 create_thread(rcu_read_stress_test, NULL);
 528         create_thread(rcu_update_stress_test, NULL);
 529         for (i = 0; i < 5; i++)
 530                 create_thread(rcu_fake_update_stress_test, NULL);
 531         cmm_smp_mb();
 532         goflag = GOFLAG_RUN;
 533         cmm_smp_mb();
 534         sleep(10);
 535         cmm_smp_mb();
 536         goflag = GOFLAG_STOP;
 537         cmm_smp_mb();
 538         wait_all_threads();
 539         for_each_thread(t)
 540                 n_reads += per_thread(n_reads_pt, t);
 541         diag("n_reads: %lld  n_updates: %ld  n_mberror: %d",
 542                n_reads, n_updates, n_mberror);
 543         rdiag_start();
 544         rdiag("rcu_stress_count:");
 545         for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
 546                 sum = 0LL;
 547                 for_each_thread(t) {
 548                         sum += per_thread(rcu_stress_count, t)[i];
 549                 }
 550                 rdiag(" %lld", sum);
 551         }
 552         rdiag_end();
 553         if (get_cpu_call_rcu_data(0)) {
 554                 diag("Deallocating per-CPU call_rcu threads.");
 555                 free_all_cpu_call_rcu_data();
 556         }
 557         if (!n_mberror)
 558                 return 0;
 559         else
 560                 return -1;
 561 }
 562
 563 /*
 564  * Mainprogram.
 565  */
 566
 567 static
 568 void usage(char *argv[]) __attribute__((__noreturn__));
 569
 570 static
 571 void usage(char *argv[])
 572 {
 573         diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
 574         exit(-1);
 575 }
 576
 577 int main(int argc, char *argv[])
 578 {
 579         int nreaders = 1;
 580         int cpustride = 1;
 581
 582         plan_tests(NR_TESTS);
 583
 584         smp_init();
 585         //rcu_init();
 586         if (argc > 4) {
 587                 const char *callrcu_str = argv[4];;
 588
 589                 if (strcmp(callrcu_str, "callrcu_global") == 0) {
 590                         callrcu_type = CALLRCU_GLOBAL;
 591                 } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
 592                         callrcu_type = CALLRCU_PERCPU;
 593                 } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
 594                         callrcu_type = CALLRCU_PERTHREAD;
 595                 } else {
 596                         usage(argv);
 597                         goto end;
 598                 }
 599         }
 600
 601         switch (callrcu_type) {
 602         case CALLRCU_GLOBAL:
 603                 diag("Using global per-process call_rcu thread.");
 604                 break;
 605         case CALLRCU_PERCPU:
 606                 diag("Using per-CPU call_rcu threads.");
 607                 if (create_all_cpu_call_rcu_data(0))
 608                         diag("create_all_cpu_call_rcu_data: %s",
 609                                 strerror(errno));
 610                 break;
 611         case CALLRCU_PERTHREAD:
 612                 diag("Using per-thread call_rcu() worker.");
 613                 break;
 614         default:
 615                 abort();
 616         }
 617
 618 #ifdef DEBUG_YIELD
 619         yield_active |= YIELD_READ;
 620         yield_active |= YIELD_WRITE;
 621 #endif
 622
 623         if (argc > 1) {
 624                 if (strcmp(argv[1], "-h") == 0
 625                                 || strcmp(argv[1], "--help") == 0) {
 626                         usage(argv);
 627                         goto end;
 628                 }
 629                 nreaders = strtoul(argv[1], NULL, 0);
 630                 if (argc == 2) {
 631                         ok(!perftest(nreaders, cpustride),
 632                                 "perftest readers: %d, stride: %d",
 633                                 nreaders, cpustride);
 634                         goto end;
 635                 }
 636                 if (argc > 3)
 637                         cpustride = strtoul(argv[3], NULL, 0);
 638                 if (strcmp(argv[2], "perf") == 0)
 639                         ok(!perftest(nreaders, cpustride),
 640                                 "perftest readers: %d, stride: %d",
 641                                 nreaders, cpustride);
 642                 else if (strcmp(argv[2], "rperf") == 0)
 643                         ok(!rperftest(nreaders, cpustride),
 644                                 "rperftest readers: %d, stride: %d",
 645                                 nreaders, cpustride);
 646                 else if (strcmp(argv[2], "uperf") == 0)
 647                         ok(!uperftest(nreaders, cpustride),
 648                                 "uperftest readers: %d, stride: %d",
 649                                 nreaders, cpustride);
 650                 else if (strcmp(argv[2], "stress") == 0)
 651                         ok(!stresstest(nreaders),
 652                                 "stresstest readers: %d, stride: %d",
 653                                 nreaders, cpustride);
 654                 else
 655                         usage(argv);
 656         } else {
 657                 usage(argv);
 658         }
 659 end:
 660         return exit_status();
 661 }