tests/regression/rcutorture.h

   1 /*
   2  * rcutorture.h: simple user-level performance/stress test of RCU.
   3  *
   4  * Usage:
   5  *      ./rcu <nreaders> rperf [ <cpustride> ]
   6  *              Run a read-side performance test with the specified
   7  *              number of readers spaced by <cpustride>.
   8  *              Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
   9  *              CPUs from 0 to 30.
  10  *      ./rcu <nupdaters> uperf [ <cpustride> ]
  11  *              Run an update-side performance test with the specified
  12  *              number of updaters and specified CPU spacing.
  13  *      ./rcu <nreaders> perf [ <cpustride> ]
  14  *              Run a combined read/update performance test with the specified
  15  *              number of readers and one updater and specified CPU spacing.
  16  *              The readers run on the low-numbered CPUs and the updater
  17  *              of the highest-numbered CPU.
  18  *
  19  * The above tests produce output as follows:
  20  *
  21  * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
  22  * ns/read: 43.4707  ns/update: 6848.1
  23  *
  24  * The first line lists the total number of RCU reads and updates executed
  25  * during the test, the number of reader threads, the number of updater
  26  * threads, and the duration of the test in seconds.  The second line
  27  * lists the average duration of each type of operation in nanoseconds,
  28  * or "nan" if the corresponding type of operation was not performed.
  29  *
  30  *      ./rcu <nreaders> stress
  31  *              Run a stress test with the specified number of readers and
  32  *              one updater.  None of the threads are affinitied to any
  33  *              particular CPU.
  34  *
  35  * This test produces output as follows:
  36  *
  37  * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
  38  * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
  39  *
  40  * The first line lists the number of RCU read and update operations
  41  * executed, followed by the number of memory-ordering violations
  42  * (which will be zero in a correct RCU implementation).  The second
  43  * line lists the number of readers observing progressively more stale
  44  * data.  A correct RCU implementation will have all but the first two
  45  * numbers non-zero.
  46  *
  47  * This program is free software; you can redistribute it and/or modify
  48  * it under the terms of the GNU General Public License as published by
  49  * the Free Software Foundation; either version 2 of the License, or
  50  * (at your option) any later version.
  51  *
  52  * This program is distributed in the hope that it will be useful,
  53  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  54  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  55  * GNU General Public License for more details.
  56  *
  57  * You should have received a copy of the GNU General Public License
  58  * along with this program; if not, write to the Free Software
  59  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  60  *
  61  * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
  62  */
  63
  64 /*
  65  * Test variables.
  66  */
  67
  68 #include <stdlib.h>
  69 #include "tap.h"
  70
  71 #define NR_TESTS        1
  72
  73 DEFINE_PER_THREAD(long long, n_reads_pt);
  74 DEFINE_PER_THREAD(long long, n_updates_pt);
  75
  76 long long n_reads = 0LL;
  77 long n_updates = 0L;
  78 int nthreadsrunning;
  79 char argsbuf[64];
  80
  81 #define GOFLAG_INIT 0
  82 #define GOFLAG_RUN  1
  83 #define GOFLAG_STOP 2
  84
  85 volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
  86         = GOFLAG_INIT;
  87
  88 #define RCU_READ_RUN 1000
  89
  90 //MD
  91 #define RCU_READ_NESTABLE
  92
  93 #ifdef RCU_READ_NESTABLE
  94 #define rcu_read_lock_nest() rcu_read_lock()
  95 #define rcu_read_unlock_nest() rcu_read_unlock()
  96 #else /* #ifdef RCU_READ_NESTABLE */
  97 #define rcu_read_lock_nest()
  98 #define rcu_read_unlock_nest()
  99 #endif /* #else #ifdef RCU_READ_NESTABLE */
 100
 101 #ifdef TORTURE_QSBR
 102 #define mark_rcu_quiescent_state        rcu_quiescent_state
 103 #define put_thread_offline              rcu_thread_offline
 104 #define put_thread_online               rcu_thread_online
 105 #endif
 106
 107 #ifndef mark_rcu_quiescent_state
 108 #define mark_rcu_quiescent_state() do ; while (0)
 109 #endif /* #ifdef mark_rcu_quiescent_state */
 110
 111 #ifndef put_thread_offline
 112 #define put_thread_offline()            do ; while (0)
 113 #define put_thread_online()             do ; while (0)
 114 #define put_thread_online_delay()       do ; while (0)
 115 #else /* #ifndef put_thread_offline */
 116 #define put_thread_online_delay()       synchronize_rcu()
 117 #endif /* #else #ifndef put_thread_offline */
 118
 119 /*
 120  * Performance test.
 121  */
 122
 123 void *rcu_read_perf_test(void *arg)
 124 {
 125         struct call_rcu_data *crdp;
 126         int i;
 127         int me = (long)arg;
 128         long long n_reads_local = 0;
 129
 130         rcu_register_thread();
 131         run_on(me);
 132         uatomic_inc(&nthreadsrunning);
 133         put_thread_offline();
 134         while (goflag == GOFLAG_INIT)
 135                 (void) poll(NULL, 0, 1);
 136         put_thread_online();
 137         while (goflag == GOFLAG_RUN) {
 138                 for (i = 0; i < RCU_READ_RUN; i++) {
 139                         rcu_read_lock();
 140                         /* rcu_read_lock_nest(); */
 141                         /* rcu_read_unlock_nest(); */
 142                         rcu_read_unlock();
 143                 }
 144                 n_reads_local += RCU_READ_RUN;
 145                 mark_rcu_quiescent_state();
 146         }
 147         __get_thread_var(n_reads_pt) += n_reads_local;
 148         put_thread_offline();
 149         crdp = get_thread_call_rcu_data();
 150         set_thread_call_rcu_data(NULL);
 151         call_rcu_data_free(crdp);
 152         rcu_unregister_thread();
 153
 154         return (NULL);
 155 }
 156
 157 void *rcu_update_perf_test(void *arg)
 158 {
 159         long long n_updates_local = 0;
 160
 161         if ((random() & 0xf00) == 0) {
 162                 struct call_rcu_data *crdp;
 163
 164                 crdp = create_call_rcu_data(0, -1);
 165                 if (crdp != NULL) {
 166                         diag("Using per-thread call_rcu() worker.");
 167                         set_thread_call_rcu_data(crdp);
 168                 }
 169         }
 170         uatomic_inc(&nthreadsrunning);
 171         while (goflag == GOFLAG_INIT)
 172                 (void) poll(NULL, 0, 1);
 173         while (goflag == GOFLAG_RUN) {
 174                 synchronize_rcu();
 175                 n_updates_local++;
 176         }
 177         __get_thread_var(n_updates_pt) += n_updates_local;
 178         return NULL;
 179 }
 180
 181 void perftestinit(void)
 182 {
 183         init_per_thread(n_reads_pt, 0LL);
 184         init_per_thread(n_updates_pt, 0LL);
 185         uatomic_set(&nthreadsrunning, 0);
 186 }
 187
 188 int perftestrun(int nthreads, int nreaders, int nupdaters)
 189 {
 190         int t;
 191         int duration = 1;
 192
 193         cmm_smp_mb();
 194         while (uatomic_read(&nthreadsrunning) < nthreads)
 195                 (void) poll(NULL, 0, 1);
 196         goflag = GOFLAG_RUN;
 197         cmm_smp_mb();
 198         sleep(duration);
 199         cmm_smp_mb();
 200         goflag = GOFLAG_STOP;
 201         cmm_smp_mb();
 202         wait_all_threads();
 203         for_each_thread(t) {
 204                 n_reads += per_thread(n_reads_pt, t);
 205                 n_updates += per_thread(n_updates_pt, t);
 206         }
 207         diag("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d",
 208                n_reads, n_updates, nreaders, nupdaters, duration);
 209         diag("ns/read: %g  ns/update: %g",
 210                ((duration * 1000*1000*1000.*(double)nreaders) /
 211                 (double)n_reads),
 212                ((duration * 1000*1000*1000.*(double)nupdaters) /
 213                 (double)n_updates));
 214         if (get_cpu_call_rcu_data(0)) {
 215                 diag("Deallocating per-CPU call_rcu threads.\n");
 216                 free_all_cpu_call_rcu_data();
 217         }
 218         return 0;
 219 }
 220
 221 int perftest(int nreaders, int cpustride)
 222 {
 223         int i;
 224         long arg;
 225
 226         perftestinit();
 227         for (i = 0; i < nreaders; i++) {
 228                 arg = (long)(i * cpustride);
 229                 create_thread(rcu_read_perf_test, (void *)arg);
 230         }
 231         arg = (long)(i * cpustride);
 232         create_thread(rcu_update_perf_test, (void *)arg);
 233         return perftestrun(i + 1, nreaders, 1);
 234 }
 235
 236 int rperftest(int nreaders, int cpustride)
 237 {
 238         int i;
 239         long arg;
 240
 241         perftestinit();
 242         init_per_thread(n_reads_pt, 0LL);
 243         for (i = 0; i < nreaders; i++) {
 244                 arg = (long)(i * cpustride);
 245                 create_thread(rcu_read_perf_test, (void *)arg);
 246         }
 247         return perftestrun(i, nreaders, 0);
 248 }
 249
 250 int uperftest(int nupdaters, int cpustride)
 251 {
 252         int i;
 253         long arg;
 254
 255         perftestinit();
 256         init_per_thread(n_reads_pt, 0LL);
 257         for (i = 0; i < nupdaters; i++) {
 258                 arg = (long)(i * cpustride);
 259                 create_thread(rcu_update_perf_test, (void *)arg);
 260         }
 261         return perftestrun(i, 0, nupdaters);
 262 }
 263
 264 /*
 265  * Stress test.
 266  */
 267
 268 #define RCU_STRESS_PIPE_LEN 10
 269
 270 struct rcu_stress {
 271         int pipe_count;
 272         int mbtest;
 273 };
 274
 275 struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
 276 struct rcu_stress *rcu_stress_current;
 277 int rcu_stress_idx = 0;
 278
 279 int n_mberror = 0;
 280 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
 281
 282 int garbage = 0;
 283
 284 void *rcu_read_stress_test(void *arg)
 285 {
 286         int i;
 287         int itercnt = 0;
 288         struct rcu_stress *p;
 289         int pc;
 290
 291         rcu_register_thread();
 292         put_thread_offline();
 293         while (goflag == GOFLAG_INIT)
 294                 (void) poll(NULL, 0, 1);
 295         put_thread_online();
 296         while (goflag == GOFLAG_RUN) {
 297                 rcu_read_lock();
 298                 p = rcu_dereference(rcu_stress_current);
 299                 if (p->mbtest == 0)
 300                         n_mberror++;
 301                 rcu_read_lock_nest();
 302                 for (i = 0; i < 100; i++)
 303                         garbage++;
 304                 rcu_read_unlock_nest();
 305                 pc = p->pipe_count;
 306                 rcu_read_unlock();
 307                 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
 308                         pc = RCU_STRESS_PIPE_LEN;
 309                 __get_thread_var(rcu_stress_count)[pc]++;
 310                 __get_thread_var(n_reads_pt)++;
 311                 mark_rcu_quiescent_state();
 312                 if ((++itercnt % 0x1000) == 0) {
 313                         put_thread_offline();
 314                         put_thread_online_delay();
 315                         put_thread_online();
 316                 }
 317         }
 318         put_thread_offline();
 319         rcu_unregister_thread();
 320
 321         return (NULL);
 322 }
 323
 324 static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
 325 static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
 326
 327 void rcu_update_stress_test_rcu(struct rcu_head *head)
 328 {
 329         int ret;
 330
 331         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 332         if (ret) {
 333                 errno = ret;
 334                 diag("pthread_mutex_lock: %s",
 335                         strerror(errno));
 336                 abort();
 337         }
 338         ret = pthread_cond_signal(&call_rcu_test_cond);
 339         if (ret) {
 340                 errno = ret;
 341                 diag("pthread_cond_signal: %s",
 342                         strerror(errno));
 343                 abort();
 344         }
 345         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 346         if (ret) {
 347                 errno = ret;
 348                 diag("pthread_mutex_unlock: %s",
 349                         strerror(errno));
 350                 abort();
 351         }
 352 }
 353
 354 void *rcu_update_stress_test(void *arg)
 355 {
 356         int i;
 357         struct rcu_stress *p;
 358         struct rcu_head rh;
 359
 360         while (goflag == GOFLAG_INIT)
 361                 (void) poll(NULL, 0, 1);
 362         while (goflag == GOFLAG_RUN) {
 363                 i = rcu_stress_idx + 1;
 364                 if (i >= RCU_STRESS_PIPE_LEN)
 365                         i = 0;
 366                 p = &rcu_stress_array[i];
 367                 p->mbtest = 0;
 368                 cmm_smp_mb();
 369                 p->pipe_count = 0;
 370                 p->mbtest = 1;
 371                 rcu_assign_pointer(rcu_stress_current, p);
 372                 rcu_stress_idx = i;
 373                 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
 374                         if (i != rcu_stress_idx)
 375                                 rcu_stress_array[i].pipe_count++;
 376                 if (n_updates & 0x1)
 377                         synchronize_rcu();
 378                 else {
 379                         int ret;
 380
 381                         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 382                         if (ret) {
 383                                 errno = ret;
 384                                 diag("pthread_mutex_lock: %s",
 385                                         strerror(errno));
 386                                 abort();
 387                         }
 388                         call_rcu(&rh, rcu_update_stress_test_rcu);
 389                         ret = pthread_cond_wait(&call_rcu_test_cond,
 390                                         &call_rcu_test_mutex);
 391                         if (ret) {
 392                                 errno = ret;
 393                                 diag("pthread_cond_signal: %s",
 394                                         strerror(errno));
 395                                 abort();
 396                         }
 397                         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 398                         if (ret) {
 399                                 errno = ret;
 400                                 diag("pthread_mutex_unlock: %s",
 401                                         strerror(errno));
 402                                 abort();
 403                         }
 404                 }
 405                 n_updates++;
 406         }
 407         return NULL;
 408 }
 409
 410 void *rcu_fake_update_stress_test(void *arg)
 411 {
 412         if ((random() & 0xf00) == 0) {
 413                 struct call_rcu_data *crdp;
 414
 415                 crdp = create_call_rcu_data(0, -1);
 416                 if (crdp != NULL) {
 417                         diag("Using per-thread call_rcu() worker.");
 418                         set_thread_call_rcu_data(crdp);
 419                 }
 420         }
 421         while (goflag == GOFLAG_INIT)
 422                 (void) poll(NULL, 0, 1);
 423         while (goflag == GOFLAG_RUN) {
 424                 synchronize_rcu();
 425                 (void) poll(NULL, 0, 1);
 426         }
 427         return NULL;
 428 }
 429
 430 int stresstest(int nreaders)
 431 {
 432         int i;
 433         int t;
 434         long long *p;
 435         long long sum;
 436
 437         init_per_thread(n_reads_pt, 0LL);
 438         for_each_thread(t) {
 439                 p = &per_thread(rcu_stress_count,t)[0];
 440                 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
 441                         p[i] = 0LL;
 442         }
 443         rcu_stress_current = &rcu_stress_array[0];
 444         rcu_stress_current->pipe_count = 0;
 445         rcu_stress_current->mbtest = 1;
 446         for (i = 0; i < nreaders; i++)
 447                 create_thread(rcu_read_stress_test, NULL);
 448         create_thread(rcu_update_stress_test, NULL);
 449         for (i = 0; i < 5; i++)
 450                 create_thread(rcu_fake_update_stress_test, NULL);
 451         cmm_smp_mb();
 452         goflag = GOFLAG_RUN;
 453         cmm_smp_mb();
 454         sleep(10);
 455         cmm_smp_mb();
 456         goflag = GOFLAG_STOP;
 457         cmm_smp_mb();
 458         wait_all_threads();
 459         for_each_thread(t)
 460                 n_reads += per_thread(n_reads_pt, t);
 461         diag("n_reads: %lld  n_updates: %ld  n_mberror: %d",
 462                n_reads, n_updates, n_mberror);
 463         rdiag_start();
 464         rdiag("rcu_stress_count:");
 465         for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
 466                 sum = 0LL;
 467                 for_each_thread(t) {
 468                         sum += per_thread(rcu_stress_count, t)[i];
 469                 }
 470                 rdiag(" %lld", sum);
 471         }
 472         rdiag_end();
 473         if (get_cpu_call_rcu_data(0)) {
 474                 diag("Deallocating per-CPU call_rcu threads.");
 475                 free_all_cpu_call_rcu_data();
 476         }
 477         if (!n_mberror)
 478                 return 0;
 479         else
 480                 return -1;
 481 }
 482
 483 /*
 484  * Mainprogram.
 485  */
 486
 487 void usage(int argc, char *argv[])
 488 {
 489         diag("Usage: %s [nreaders [ perf | rperf | uperf | stress ] ]\n", argv[0]);
 490         exit(-1);
 491 }
 492
 493 int main(int argc, char *argv[])
 494 {
 495         int nreaders = 1;
 496         int cpustride = 1;
 497
 498         plan_tests(NR_TESTS);
 499
 500         smp_init();
 501         //rcu_init();
 502         srandom(time(NULL));
 503         if (random() & 0x100) {
 504                 diag("Allocating per-CPU call_rcu threads.");
 505                 if (create_all_cpu_call_rcu_data(0))
 506                         diag("create_all_cpu_call_rcu_data: %s",
 507                                 strerror(errno));
 508         }
 509
 510 #ifdef DEBUG_YIELD
 511         yield_active |= YIELD_READ;
 512         yield_active |= YIELD_WRITE;
 513 #endif
 514
 515         if (argc > 1) {
 516                 nreaders = strtoul(argv[1], NULL, 0);
 517                 if (argc == 2) {
 518                         ok(!perftest(nreaders, cpustride),
 519                                 "perftest readers: %d, stride: %d",
 520                                 nreaders, cpustride);
 521                         goto end;
 522                 }
 523                 if (argc > 3)
 524                         cpustride = strtoul(argv[3], NULL, 0);
 525                 if (strcmp(argv[2], "perf") == 0)
 526                         ok(!perftest(nreaders, cpustride),
 527                                 "perftest readers: %d, stride: %d",
 528                                 nreaders, cpustride);
 529                 else if (strcmp(argv[2], "rperf") == 0)
 530                         ok(!rperftest(nreaders, cpustride),
 531                                 "rperftest readers: %d, stride: %d",
 532                                 nreaders, cpustride);
 533                 else if (strcmp(argv[2], "uperf") == 0)
 534                         ok(!uperftest(nreaders, cpustride),
 535                                 "uperftest readers: %d, stride: %d",
 536                                 nreaders, cpustride);
 537                 else if (strcmp(argv[2], "stress") == 0)
 538                         ok(!stresstest(nreaders),
 539                                 "stresstest readers: %d, stride: %d",
 540                                 nreaders, cpustride);
 541                 else
 542                         usage(argc, argv);
 543         } else {
 544                 ok(!perftest(nreaders, cpustride),
 545                         "perftest readers: %d, stride: %d",
 546                         nreaders, cpustride);
 547         }
 548 end:
 549         return exit_status();
 550 }