fix: shadowed local variable (-Wshadow)
[urcu.git] / tests / regression / rcutorture.h
CommitLineData
8a953620
MD
1/*
2 * rcutorture.h: simple user-level performance/stress test of RCU.
3 *
4 * Usage:
5 * ./rcu <nreaders> rperf [ <cpustride> ]
6 * Run a read-side performance test with the specified
7 * number of readers spaced by <cpustride>.
8 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
9 * CPUs from 0 to 30.
10 * ./rcu <nupdaters> uperf [ <cpustride> ]
11 * Run an update-side performance test with the specified
12 * number of updaters and specified CPU spacing.
13 * ./rcu <nreaders> perf [ <cpustride> ]
14 * Run a combined read/update performance test with the specified
15 * number of readers and one updater and specified CPU spacing.
16 * The readers run on the low-numbered CPUs and the updater
17 * of the highest-numbered CPU.
18 *
19 * The above tests produce output as follows:
20 *
21 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
22 * ns/read: 43.4707 ns/update: 6848.1
23 *
24 * The first line lists the total number of RCU reads and updates executed
25 * during the test, the number of reader threads, the number of updater
26 * threads, and the duration of the test in seconds. The second line
27 * lists the average duration of each type of operation in nanoseconds,
28 * or "nan" if the corresponding type of operation was not performed.
29 *
30 * ./rcu <nreaders> stress
31 * Run a stress test with the specified number of readers and
32 * one updater. None of the threads are affinitied to any
33 * particular CPU.
34 *
35 * This test produces output as follows:
36 *
37 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
38 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
39 *
40 * The first line lists the number of RCU read and update operations
41 * executed, followed by the number of memory-ordering violations
42 * (which will be zero in a correct RCU implementation). The second
43 * line lists the number of readers observing progressively more stale
44 * data. A correct RCU implementation will have all but the first two
45 * numbers non-zero.
46 *
47 * This program is free software; you can redistribute it and/or modify
48 * it under the terms of the GNU General Public License as published by
49 * the Free Software Foundation; either version 2 of the License, or
50 * (at your option) any later version.
51 *
52 * This program is distributed in the hope that it will be useful,
53 * but WITHOUT ANY WARRANTY; without even the implied warranty of
54 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
55 * GNU General Public License for more details.
56 *
57 * You should have received a copy of the GNU General Public License
58 * along with this program; if not, write to the Free Software
3282a76b 59 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
8a953620
MD
60 *
61 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
62 */
63
64/*
65 * Test variables.
66 */
67
b57aee66 68#include <stdlib.h>
ad460058
MD
69#include "tap.h"
70
71#define NR_TESTS 1
b57aee66 72
8a953620
MD
73DEFINE_PER_THREAD(long long, n_reads_pt);
74DEFINE_PER_THREAD(long long, n_updates_pt);
75
26b5a74b
MD
76enum callrcu_type {
77 CALLRCU_GLOBAL,
78 CALLRCU_PERCPU,
79 CALLRCU_PERTHREAD,
80};
81
82static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
83
8a953620
MD
84long long n_reads = 0LL;
85long n_updates = 0L;
6ee91d83 86int nthreadsrunning;
8a953620
MD
87char argsbuf[64];
88
89#define GOFLAG_INIT 0
90#define GOFLAG_RUN 1
91#define GOFLAG_STOP 2
92
4967f005
PB
93volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
94 = GOFLAG_INIT;
8a953620
MD
95
96#define RCU_READ_RUN 1000
97
98//MD
99#define RCU_READ_NESTABLE
100
101#ifdef RCU_READ_NESTABLE
102#define rcu_read_lock_nest() rcu_read_lock()
103#define rcu_read_unlock_nest() rcu_read_unlock()
104#else /* #ifdef RCU_READ_NESTABLE */
105#define rcu_read_lock_nest()
106#define rcu_read_unlock_nest()
107#endif /* #else #ifdef RCU_READ_NESTABLE */
108
1a43bbd8
MD
109#ifdef TORTURE_QSBR
110#define mark_rcu_quiescent_state rcu_quiescent_state
111#define put_thread_offline rcu_thread_offline
112#define put_thread_online rcu_thread_online
113#endif
114
8a953620 115#ifndef mark_rcu_quiescent_state
447c9339 116#define mark_rcu_quiescent_state() do {} while (0)
8a953620
MD
117#endif /* #ifdef mark_rcu_quiescent_state */
118
119#ifndef put_thread_offline
447c9339
MJ
120#define put_thread_offline() do {} while (0)
121#define put_thread_online() do {} while (0)
122#define put_thread_online_delay() do {} while (0)
8a953620
MD
123#else /* #ifndef put_thread_offline */
124#define put_thread_online_delay() synchronize_rcu()
125#endif /* #else #ifndef put_thread_offline */
126
127/*
128 * Performance test.
129 */
130
61c3fb60 131static
8a953620
MD
132void *rcu_read_perf_test(void *arg)
133{
134 int i;
135 int me = (long)arg;
8a953620
MD
136 long long n_reads_local = 0;
137
121a5d44 138 rcu_register_thread();
8a953620 139 run_on(me);
ec4e58a3 140 uatomic_inc(&nthreadsrunning);
3bf02b5b 141 put_thread_offline();
8a953620 142 while (goflag == GOFLAG_INIT)
775aff2e 143 (void) poll(NULL, 0, 1);
3bf02b5b 144 put_thread_online();
8a953620
MD
145 while (goflag == GOFLAG_RUN) {
146 for (i = 0; i < RCU_READ_RUN; i++) {
147 rcu_read_lock();
148 /* rcu_read_lock_nest(); */
149 /* rcu_read_unlock_nest(); */
150 rcu_read_unlock();
151 }
152 n_reads_local += RCU_READ_RUN;
153 mark_rcu_quiescent_state();
154 }
155 __get_thread_var(n_reads_pt) += n_reads_local;
156 put_thread_offline();
121a5d44 157 rcu_unregister_thread();
8a953620
MD
158
159 return (NULL);
160}
161
61c3fb60 162static
8a953620
MD
163void *rcu_update_perf_test(void *arg)
164{
165 long long n_updates_local = 0;
166
26b5a74b 167 if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66
PM
168 struct call_rcu_data *crdp;
169
c1d2c60b 170 crdp = create_call_rcu_data(0, -1);
b57aee66 171 if (crdp != NULL) {
26b5a74b 172 diag("Successfully using per-thread call_rcu() worker.");
b57aee66
PM
173 set_thread_call_rcu_data(crdp);
174 }
175 }
ec4e58a3 176 uatomic_inc(&nthreadsrunning);
8a953620 177 while (goflag == GOFLAG_INIT)
775aff2e 178 (void) poll(NULL, 0, 1);
8a953620
MD
179 while (goflag == GOFLAG_RUN) {
180 synchronize_rcu();
181 n_updates_local++;
182 }
183 __get_thread_var(n_updates_pt) += n_updates_local;
26b5a74b
MD
184 if (callrcu_type == CALLRCU_PERTHREAD) {
185 struct call_rcu_data *crdp;
186
187 crdp = get_thread_call_rcu_data();
188 set_thread_call_rcu_data(NULL);
189 call_rcu_data_free(crdp);
190 }
b0b31506 191 return NULL;
8a953620
MD
192}
193
61c3fb60 194static
8a953620
MD
195void perftestinit(void)
196{
197 init_per_thread(n_reads_pt, 0LL);
198 init_per_thread(n_updates_pt, 0LL);
ec4e58a3 199 uatomic_set(&nthreadsrunning, 0);
8a953620
MD
200}
201
61c3fb60 202static
ad460058 203int perftestrun(int nthreads, int nreaders, int nupdaters)
8a953620
MD
204{
205 int t;
206 int duration = 1;
207
5481ddb3 208 cmm_smp_mb();
ec4e58a3 209 while (uatomic_read(&nthreadsrunning) < nthreads)
775aff2e 210 (void) poll(NULL, 0, 1);
8a953620 211 goflag = GOFLAG_RUN;
5481ddb3 212 cmm_smp_mb();
8a953620 213 sleep(duration);
5481ddb3 214 cmm_smp_mb();
8a953620 215 goflag = GOFLAG_STOP;
5481ddb3 216 cmm_smp_mb();
8a953620
MD
217 wait_all_threads();
218 for_each_thread(t) {
219 n_reads += per_thread(n_reads_pt, t);
220 n_updates += per_thread(n_updates_pt, t);
221 }
ad460058 222 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
8a953620 223 n_reads, n_updates, nreaders, nupdaters, duration);
ad460058 224 diag("ns/read: %g ns/update: %g",
8a953620
MD
225 ((duration * 1000*1000*1000.*(double)nreaders) /
226 (double)n_reads),
227 ((duration * 1000*1000*1000.*(double)nupdaters) /
228 (double)n_updates));
7106ddf8 229 if (get_cpu_call_rcu_data(0)) {
ad460058 230 diag("Deallocating per-CPU call_rcu threads.\n");
7106ddf8
PM
231 free_all_cpu_call_rcu_data();
232 }
ad460058 233 return 0;
8a953620
MD
234}
235
61c3fb60 236static
ad460058 237int perftest(int nreaders, int cpustride)
8a953620
MD
238{
239 int i;
240 long arg;
241
242 perftestinit();
243 for (i = 0; i < nreaders; i++) {
244 arg = (long)(i * cpustride);
245 create_thread(rcu_read_perf_test, (void *)arg);
246 }
247 arg = (long)(i * cpustride);
248 create_thread(rcu_update_perf_test, (void *)arg);
ad460058 249 return perftestrun(i + 1, nreaders, 1);
8a953620
MD
250}
251
61c3fb60 252static
ad460058 253int rperftest(int nreaders, int cpustride)
8a953620
MD
254{
255 int i;
256 long arg;
257
258 perftestinit();
259 init_per_thread(n_reads_pt, 0LL);
260 for (i = 0; i < nreaders; i++) {
261 arg = (long)(i * cpustride);
262 create_thread(rcu_read_perf_test, (void *)arg);
263 }
ad460058 264 return perftestrun(i, nreaders, 0);
8a953620
MD
265}
266
61c3fb60 267static
ad460058 268int uperftest(int nupdaters, int cpustride)
8a953620
MD
269{
270 int i;
271 long arg;
272
273 perftestinit();
274 init_per_thread(n_reads_pt, 0LL);
275 for (i = 0; i < nupdaters; i++) {
276 arg = (long)(i * cpustride);
277 create_thread(rcu_update_perf_test, (void *)arg);
278 }
ad460058 279 return perftestrun(i, 0, nupdaters);
8a953620
MD
280}
281
282/*
283 * Stress test.
284 */
285
286#define RCU_STRESS_PIPE_LEN 10
287
288struct rcu_stress {
289 int pipe_count;
290 int mbtest;
291};
292
b0b31506 293struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
8a953620
MD
294struct rcu_stress *rcu_stress_current;
295int rcu_stress_idx = 0;
296
297int n_mberror = 0;
298DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
299
300int garbage = 0;
301
61c3fb60 302static
8a953620
MD
303void *rcu_read_stress_test(void *arg)
304{
305 int i;
306 int itercnt = 0;
307 struct rcu_stress *p;
308 int pc;
309
121a5d44 310 rcu_register_thread();
3bf02b5b 311 put_thread_offline();
8a953620 312 while (goflag == GOFLAG_INIT)
775aff2e 313 (void) poll(NULL, 0, 1);
3bf02b5b 314 put_thread_online();
8a953620
MD
315 while (goflag == GOFLAG_RUN) {
316 rcu_read_lock();
317 p = rcu_dereference(rcu_stress_current);
318 if (p->mbtest == 0)
319 n_mberror++;
320 rcu_read_lock_nest();
321 for (i = 0; i < 100; i++)
322 garbage++;
323 rcu_read_unlock_nest();
324 pc = p->pipe_count;
325 rcu_read_unlock();
326 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
327 pc = RCU_STRESS_PIPE_LEN;
328 __get_thread_var(rcu_stress_count)[pc]++;
329 __get_thread_var(n_reads_pt)++;
330 mark_rcu_quiescent_state();
331 if ((++itercnt % 0x1000) == 0) {
332 put_thread_offline();
333 put_thread_online_delay();
334 put_thread_online();
335 }
336 }
337 put_thread_offline();
121a5d44 338 rcu_unregister_thread();
8a953620
MD
339
340 return (NULL);
341}
342
b57aee66
PM
343static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
344static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
345
61c3fb60 346static
b57aee66
PM
347void rcu_update_stress_test_rcu(struct rcu_head *head)
348{
ad460058
MD
349 int ret;
350
351 ret = pthread_mutex_lock(&call_rcu_test_mutex);
352 if (ret) {
353 errno = ret;
354 diag("pthread_mutex_lock: %s",
355 strerror(errno));
356 abort();
b57aee66 357 }
ad460058
MD
358 ret = pthread_cond_signal(&call_rcu_test_cond);
359 if (ret) {
360 errno = ret;
361 diag("pthread_cond_signal: %s",
362 strerror(errno));
363 abort();
b57aee66 364 }
ad460058
MD
365 ret = pthread_mutex_unlock(&call_rcu_test_mutex);
366 if (ret) {
367 errno = ret;
368 diag("pthread_mutex_unlock: %s",
369 strerror(errno));
370 abort();
b57aee66
PM
371 }
372}
373
61c3fb60 374static
8a953620
MD
375void *rcu_update_stress_test(void *arg)
376{
377 int i;
378 struct rcu_stress *p;
b57aee66 379 struct rcu_head rh;
8a953620
MD
380
381 while (goflag == GOFLAG_INIT)
775aff2e 382 (void) poll(NULL, 0, 1);
8a953620
MD
383 while (goflag == GOFLAG_RUN) {
384 i = rcu_stress_idx + 1;
385 if (i >= RCU_STRESS_PIPE_LEN)
386 i = 0;
387 p = &rcu_stress_array[i];
388 p->mbtest = 0;
5481ddb3 389 cmm_smp_mb();
8a953620
MD
390 p->pipe_count = 0;
391 p->mbtest = 1;
392 rcu_assign_pointer(rcu_stress_current, p);
393 rcu_stress_idx = i;
394 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
395 if (i != rcu_stress_idx)
396 rcu_stress_array[i].pipe_count++;
b57aee66
PM
397 if (n_updates & 0x1)
398 synchronize_rcu();
399 else {
ad460058
MD
400 int ret;
401
402 ret = pthread_mutex_lock(&call_rcu_test_mutex);
403 if (ret) {
404 errno = ret;
405 diag("pthread_mutex_lock: %s",
406 strerror(errno));
407 abort();
b57aee66 408 }
0b9c513b 409 rcu_register_thread();
b57aee66 410 call_rcu(&rh, rcu_update_stress_test_rcu);
0b9c513b
MD
411 rcu_unregister_thread();
412 /*
413 * Our MacOS X test machine with the following
414 * config:
415 * 15.6.0 Darwin Kernel Version 15.6.0
416 * root:xnu-3248.60.10~1/RELEASE_X86_64
417 * appears to have issues with liburcu-signal
418 * signal being delivered on top of
419 * pthread_cond_wait. It seems to make the
420 * thread continue, and therefore corrupt the
421 * rcu_head. Work around this issue by
422 * unregistering the RCU read-side thread
423 * immediately after call_rcu (call_rcu needs
424 * us to be registered RCU readers).
425 */
ad460058
MD
426 ret = pthread_cond_wait(&call_rcu_test_cond,
427 &call_rcu_test_mutex);
428 if (ret) {
429 errno = ret;
430 diag("pthread_cond_signal: %s",
431 strerror(errno));
432 abort();
b57aee66 433 }
ad460058
MD
434 ret = pthread_mutex_unlock(&call_rcu_test_mutex);
435 if (ret) {
436 errno = ret;
437 diag("pthread_mutex_unlock: %s",
438 strerror(errno));
439 abort();
b57aee66
PM
440 }
441 }
8a953620
MD
442 n_updates++;
443 }
a13ef613 444
b0b31506 445 return NULL;
8a953620
MD
446}
447
61c3fb60 448static
8a953620
MD
449void *rcu_fake_update_stress_test(void *arg)
450{
26b5a74b 451 if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66
PM
452 struct call_rcu_data *crdp;
453
c1d2c60b 454 crdp = create_call_rcu_data(0, -1);
b57aee66 455 if (crdp != NULL) {
26b5a74b 456 diag("Successfully using per-thread call_rcu() worker.");
b57aee66
PM
457 set_thread_call_rcu_data(crdp);
458 }
459 }
8a953620 460 while (goflag == GOFLAG_INIT)
775aff2e 461 (void) poll(NULL, 0, 1);
8a953620
MD
462 while (goflag == GOFLAG_RUN) {
463 synchronize_rcu();
775aff2e 464 (void) poll(NULL, 0, 1);
8a953620 465 }
26b5a74b
MD
466 if (callrcu_type == CALLRCU_PERTHREAD) {
467 struct call_rcu_data *crdp;
468
469 crdp = get_thread_call_rcu_data();
470 set_thread_call_rcu_data(NULL);
471 call_rcu_data_free(crdp);
472 }
b0b31506 473 return NULL;
8a953620
MD
474}
475
61c3fb60 476static
ad460058 477int stresstest(int nreaders)
8a953620
MD
478{
479 int i;
480 int t;
481 long long *p;
482 long long sum;
483
484 init_per_thread(n_reads_pt, 0LL);
485 for_each_thread(t) {
486 p = &per_thread(rcu_stress_count,t)[0];
487 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
488 p[i] = 0LL;
489 }
490 rcu_stress_current = &rcu_stress_array[0];
491 rcu_stress_current->pipe_count = 0;
492 rcu_stress_current->mbtest = 1;
493 for (i = 0; i < nreaders; i++)
494 create_thread(rcu_read_stress_test, NULL);
495 create_thread(rcu_update_stress_test, NULL);
496 for (i = 0; i < 5; i++)
497 create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3 498 cmm_smp_mb();
8a953620 499 goflag = GOFLAG_RUN;
5481ddb3 500 cmm_smp_mb();
8a953620 501 sleep(10);
5481ddb3 502 cmm_smp_mb();
8a953620 503 goflag = GOFLAG_STOP;
5481ddb3 504 cmm_smp_mb();
8a953620
MD
505 wait_all_threads();
506 for_each_thread(t)
507 n_reads += per_thread(n_reads_pt, t);
ad460058 508 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
8a953620 509 n_reads, n_updates, n_mberror);
ad460058
MD
510 rdiag_start();
511 rdiag("rcu_stress_count:");
8a953620
MD
512 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
513 sum = 0LL;
514 for_each_thread(t) {
515 sum += per_thread(rcu_stress_count, t)[i];
516 }
ad460058 517 rdiag(" %lld", sum);
8a953620 518 }
ad460058 519 rdiag_end();
7106ddf8 520 if (get_cpu_call_rcu_data(0)) {
ad460058 521 diag("Deallocating per-CPU call_rcu threads.");
7106ddf8
PM
522 free_all_cpu_call_rcu_data();
523 }
ad460058
MD
524 if (!n_mberror)
525 return 0;
526 else
527 return -1;
8a953620
MD
528}
529
530/*
531 * Mainprogram.
532 */
533
61c3fb60 534static
8a953620
MD
535void usage(int argc, char *argv[])
536{
26b5a74b 537 diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
8a953620
MD
538 exit(-1);
539}
540
541int main(int argc, char *argv[])
542{
543 int nreaders = 1;
544 int cpustride = 1;
545
ad460058
MD
546 plan_tests(NR_TESTS);
547
8a953620
MD
548 smp_init();
549 //rcu_init();
26b5a74b
MD
550 if (argc > 4) {
551 const char *callrcu_str = argv[4];;
552
553 if (strcmp(callrcu_str, "callrcu_global") == 0) {
554 callrcu_type = CALLRCU_GLOBAL;
555 } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
556 callrcu_type = CALLRCU_PERCPU;
557 } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
558 callrcu_type = CALLRCU_PERTHREAD;
559 } else {
560 usage(argc, argv);
561 goto end;
562 }
563 }
929cfaff 564
26b5a74b
MD
565 switch (callrcu_type) {
566 case CALLRCU_GLOBAL:
567 diag("Using global per-process call_rcu thread.");
568 break;
569 case CALLRCU_PERCPU:
570 diag("Using per-CPU call_rcu threads.");
b57aee66 571 if (create_all_cpu_call_rcu_data(0))
ad460058
MD
572 diag("create_all_cpu_call_rcu_data: %s",
573 strerror(errno));
26b5a74b
MD
574 break;
575 case CALLRCU_PERTHREAD:
576 diag("Using per-thread call_rcu() worker.");
577 break;
578 default:
579 abort();
b57aee66 580 }
8a953620 581
9b171f46
MD
582#ifdef DEBUG_YIELD
583 yield_active |= YIELD_READ;
584 yield_active |= YIELD_WRITE;
585#endif
586
8a953620 587 if (argc > 1) {
26b5a74b
MD
588 if (strcmp(argv[1], "-h") == 0
589 || strcmp(argv[1], "--help") == 0) {
590 usage(argc, argv);
591 goto end;
592 }
8a953620 593 nreaders = strtoul(argv[1], NULL, 0);
ad460058
MD
594 if (argc == 2) {
595 ok(!perftest(nreaders, cpustride),
596 "perftest readers: %d, stride: %d",
597 nreaders, cpustride);
598 goto end;
599 }
8a953620
MD
600 if (argc > 3)
601 cpustride = strtoul(argv[3], NULL, 0);
602 if (strcmp(argv[2], "perf") == 0)
ad460058
MD
603 ok(!perftest(nreaders, cpustride),
604 "perftest readers: %d, stride: %d",
605 nreaders, cpustride);
8a953620 606 else if (strcmp(argv[2], "rperf") == 0)
ad460058
MD
607 ok(!rperftest(nreaders, cpustride),
608 "rperftest readers: %d, stride: %d",
609 nreaders, cpustride);
8a953620 610 else if (strcmp(argv[2], "uperf") == 0)
ad460058
MD
611 ok(!uperftest(nreaders, cpustride),
612 "uperftest readers: %d, stride: %d",
613 nreaders, cpustride);
8a953620 614 else if (strcmp(argv[2], "stress") == 0)
ad460058
MD
615 ok(!stresstest(nreaders),
616 "stresstest readers: %d, stride: %d",
617 nreaders, cpustride);
618 else
619 usage(argc, argv);
620 } else {
26b5a74b 621 usage(argc, argv);
8a953620 622 }
ad460058
MD
623end:
624 return exit_status();
8a953620 625}
This page took 0.077059 seconds and 4 git commands to generate.