Add -Wextra to CFLAGS
[urcu.git] / tests / regression / rcutorture.h
CommitLineData
8a953620
MD
1/*
2 * rcutorture.h: simple user-level performance/stress test of RCU.
3 *
4 * Usage:
5 * ./rcu <nreaders> rperf [ <cpustride> ]
6 * Run a read-side performance test with the specified
7 * number of readers spaced by <cpustride>.
8 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
9 * CPUs from 0 to 30.
10 * ./rcu <nupdaters> uperf [ <cpustride> ]
11 * Run an update-side performance test with the specified
12 * number of updaters and specified CPU spacing.
13 * ./rcu <nreaders> perf [ <cpustride> ]
14 * Run a combined read/update performance test with the specified
15 * number of readers and one updater and specified CPU spacing.
16 * The readers run on the low-numbered CPUs and the updater
17 * of the highest-numbered CPU.
18 *
19 * The above tests produce output as follows:
20 *
21 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
22 * ns/read: 43.4707 ns/update: 6848.1
23 *
24 * The first line lists the total number of RCU reads and updates executed
25 * during the test, the number of reader threads, the number of updater
26 * threads, and the duration of the test in seconds. The second line
27 * lists the average duration of each type of operation in nanoseconds,
28 * or "nan" if the corresponding type of operation was not performed.
29 *
30 * ./rcu <nreaders> stress
31 * Run a stress test with the specified number of readers and
32 * one updater. None of the threads are affinitied to any
33 * particular CPU.
34 *
35 * This test produces output as follows:
36 *
37 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
38 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
39 *
40 * The first line lists the number of RCU read and update operations
41 * executed, followed by the number of memory-ordering violations
42 * (which will be zero in a correct RCU implementation). The second
43 * line lists the number of readers observing progressively more stale
44 * data. A correct RCU implementation will have all but the first two
45 * numbers non-zero.
46 *
47 * This program is free software; you can redistribute it and/or modify
48 * it under the terms of the GNU General Public License as published by
49 * the Free Software Foundation; either version 2 of the License, or
50 * (at your option) any later version.
51 *
52 * This program is distributed in the hope that it will be useful,
53 * but WITHOUT ANY WARRANTY; without even the implied warranty of
54 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
55 * GNU General Public License for more details.
56 *
57 * You should have received a copy of the GNU General Public License
58 * along with this program; if not, write to the Free Software
3282a76b 59 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
8a953620
MD
60 *
61 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
62 */
63
64/*
65 * Test variables.
66 */
67
b57aee66 68#include <stdlib.h>
ad460058
MD
69#include "tap.h"
70
71#define NR_TESTS 1
b57aee66 72
8a953620
MD
73DEFINE_PER_THREAD(long long, n_reads_pt);
74DEFINE_PER_THREAD(long long, n_updates_pt);
75
26b5a74b
MD
76enum callrcu_type {
77 CALLRCU_GLOBAL,
78 CALLRCU_PERCPU,
79 CALLRCU_PERTHREAD,
80};
81
82static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
83
8a953620
MD
84long long n_reads = 0LL;
85long n_updates = 0L;
6ee91d83 86int nthreadsrunning;
8a953620
MD
87char argsbuf[64];
88
89#define GOFLAG_INIT 0
90#define GOFLAG_RUN 1
91#define GOFLAG_STOP 2
92
4967f005
PB
93volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
94 = GOFLAG_INIT;
8a953620
MD
95
96#define RCU_READ_RUN 1000
97
98//MD
99#define RCU_READ_NESTABLE
100
101#ifdef RCU_READ_NESTABLE
102#define rcu_read_lock_nest() rcu_read_lock()
103#define rcu_read_unlock_nest() rcu_read_unlock()
104#else /* #ifdef RCU_READ_NESTABLE */
105#define rcu_read_lock_nest()
106#define rcu_read_unlock_nest()
107#endif /* #else #ifdef RCU_READ_NESTABLE */
108
1a43bbd8
MD
109#ifdef TORTURE_QSBR
110#define mark_rcu_quiescent_state rcu_quiescent_state
111#define put_thread_offline rcu_thread_offline
112#define put_thread_online rcu_thread_online
113#endif
114
8a953620 115#ifndef mark_rcu_quiescent_state
447c9339 116#define mark_rcu_quiescent_state() do {} while (0)
8a953620
MD
117#endif /* #ifdef mark_rcu_quiescent_state */
118
119#ifndef put_thread_offline
447c9339
MJ
120#define put_thread_offline() do {} while (0)
121#define put_thread_online() do {} while (0)
122#define put_thread_online_delay() do {} while (0)
8a953620
MD
123#else /* #ifndef put_thread_offline */
124#define put_thread_online_delay() synchronize_rcu()
125#endif /* #else #ifndef put_thread_offline */
126
127/*
128 * Performance test.
129 */
130
131void *rcu_read_perf_test(void *arg)
132{
133 int i;
134 int me = (long)arg;
8a953620
MD
135 long long n_reads_local = 0;
136
121a5d44 137 rcu_register_thread();
8a953620 138 run_on(me);
ec4e58a3 139 uatomic_inc(&nthreadsrunning);
3bf02b5b 140 put_thread_offline();
8a953620 141 while (goflag == GOFLAG_INIT)
775aff2e 142 (void) poll(NULL, 0, 1);
3bf02b5b 143 put_thread_online();
8a953620
MD
144 while (goflag == GOFLAG_RUN) {
145 for (i = 0; i < RCU_READ_RUN; i++) {
146 rcu_read_lock();
147 /* rcu_read_lock_nest(); */
148 /* rcu_read_unlock_nest(); */
149 rcu_read_unlock();
150 }
151 n_reads_local += RCU_READ_RUN;
152 mark_rcu_quiescent_state();
153 }
154 __get_thread_var(n_reads_pt) += n_reads_local;
155 put_thread_offline();
121a5d44 156 rcu_unregister_thread();
8a953620
MD
157
158 return (NULL);
159}
160
161void *rcu_update_perf_test(void *arg)
162{
163 long long n_updates_local = 0;
164
26b5a74b 165 if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66
PM
166 struct call_rcu_data *crdp;
167
c1d2c60b 168 crdp = create_call_rcu_data(0, -1);
b57aee66 169 if (crdp != NULL) {
26b5a74b 170 diag("Successfully using per-thread call_rcu() worker.");
b57aee66
PM
171 set_thread_call_rcu_data(crdp);
172 }
173 }
ec4e58a3 174 uatomic_inc(&nthreadsrunning);
8a953620 175 while (goflag == GOFLAG_INIT)
775aff2e 176 (void) poll(NULL, 0, 1);
8a953620
MD
177 while (goflag == GOFLAG_RUN) {
178 synchronize_rcu();
179 n_updates_local++;
180 }
181 __get_thread_var(n_updates_pt) += n_updates_local;
26b5a74b
MD
182 if (callrcu_type == CALLRCU_PERTHREAD) {
183 struct call_rcu_data *crdp;
184
185 crdp = get_thread_call_rcu_data();
186 set_thread_call_rcu_data(NULL);
187 call_rcu_data_free(crdp);
188 }
b0b31506 189 return NULL;
8a953620
MD
190}
191
192void perftestinit(void)
193{
194 init_per_thread(n_reads_pt, 0LL);
195 init_per_thread(n_updates_pt, 0LL);
ec4e58a3 196 uatomic_set(&nthreadsrunning, 0);
8a953620
MD
197}
198
ad460058 199int perftestrun(int nthreads, int nreaders, int nupdaters)
8a953620
MD
200{
201 int t;
202 int duration = 1;
203
5481ddb3 204 cmm_smp_mb();
ec4e58a3 205 while (uatomic_read(&nthreadsrunning) < nthreads)
775aff2e 206 (void) poll(NULL, 0, 1);
8a953620 207 goflag = GOFLAG_RUN;
5481ddb3 208 cmm_smp_mb();
8a953620 209 sleep(duration);
5481ddb3 210 cmm_smp_mb();
8a953620 211 goflag = GOFLAG_STOP;
5481ddb3 212 cmm_smp_mb();
8a953620
MD
213 wait_all_threads();
214 for_each_thread(t) {
215 n_reads += per_thread(n_reads_pt, t);
216 n_updates += per_thread(n_updates_pt, t);
217 }
ad460058 218 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
8a953620 219 n_reads, n_updates, nreaders, nupdaters, duration);
ad460058 220 diag("ns/read: %g ns/update: %g",
8a953620
MD
221 ((duration * 1000*1000*1000.*(double)nreaders) /
222 (double)n_reads),
223 ((duration * 1000*1000*1000.*(double)nupdaters) /
224 (double)n_updates));
7106ddf8 225 if (get_cpu_call_rcu_data(0)) {
ad460058 226 diag("Deallocating per-CPU call_rcu threads.\n");
7106ddf8
PM
227 free_all_cpu_call_rcu_data();
228 }
ad460058 229 return 0;
8a953620
MD
230}
231
ad460058 232int perftest(int nreaders, int cpustride)
8a953620
MD
233{
234 int i;
235 long arg;
236
237 perftestinit();
238 for (i = 0; i < nreaders; i++) {
239 arg = (long)(i * cpustride);
240 create_thread(rcu_read_perf_test, (void *)arg);
241 }
242 arg = (long)(i * cpustride);
243 create_thread(rcu_update_perf_test, (void *)arg);
ad460058 244 return perftestrun(i + 1, nreaders, 1);
8a953620
MD
245}
246
ad460058 247int rperftest(int nreaders, int cpustride)
8a953620
MD
248{
249 int i;
250 long arg;
251
252 perftestinit();
253 init_per_thread(n_reads_pt, 0LL);
254 for (i = 0; i < nreaders; i++) {
255 arg = (long)(i * cpustride);
256 create_thread(rcu_read_perf_test, (void *)arg);
257 }
ad460058 258 return perftestrun(i, nreaders, 0);
8a953620
MD
259}
260
ad460058 261int uperftest(int nupdaters, int cpustride)
8a953620
MD
262{
263 int i;
264 long arg;
265
266 perftestinit();
267 init_per_thread(n_reads_pt, 0LL);
268 for (i = 0; i < nupdaters; i++) {
269 arg = (long)(i * cpustride);
270 create_thread(rcu_update_perf_test, (void *)arg);
271 }
ad460058 272 return perftestrun(i, 0, nupdaters);
8a953620
MD
273}
274
275/*
276 * Stress test.
277 */
278
279#define RCU_STRESS_PIPE_LEN 10
280
281struct rcu_stress {
282 int pipe_count;
283 int mbtest;
284};
285
b0b31506 286struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
8a953620
MD
287struct rcu_stress *rcu_stress_current;
288int rcu_stress_idx = 0;
289
290int n_mberror = 0;
291DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
292
293int garbage = 0;
294
295void *rcu_read_stress_test(void *arg)
296{
297 int i;
298 int itercnt = 0;
299 struct rcu_stress *p;
300 int pc;
301
121a5d44 302 rcu_register_thread();
3bf02b5b 303 put_thread_offline();
8a953620 304 while (goflag == GOFLAG_INIT)
775aff2e 305 (void) poll(NULL, 0, 1);
3bf02b5b 306 put_thread_online();
8a953620
MD
307 while (goflag == GOFLAG_RUN) {
308 rcu_read_lock();
309 p = rcu_dereference(rcu_stress_current);
310 if (p->mbtest == 0)
311 n_mberror++;
312 rcu_read_lock_nest();
313 for (i = 0; i < 100; i++)
314 garbage++;
315 rcu_read_unlock_nest();
316 pc = p->pipe_count;
317 rcu_read_unlock();
318 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
319 pc = RCU_STRESS_PIPE_LEN;
320 __get_thread_var(rcu_stress_count)[pc]++;
321 __get_thread_var(n_reads_pt)++;
322 mark_rcu_quiescent_state();
323 if ((++itercnt % 0x1000) == 0) {
324 put_thread_offline();
325 put_thread_online_delay();
326 put_thread_online();
327 }
328 }
329 put_thread_offline();
121a5d44 330 rcu_unregister_thread();
8a953620
MD
331
332 return (NULL);
333}
334
b57aee66
PM
335static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
336static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
337
338void rcu_update_stress_test_rcu(struct rcu_head *head)
339{
ad460058
MD
340 int ret;
341
342 ret = pthread_mutex_lock(&call_rcu_test_mutex);
343 if (ret) {
344 errno = ret;
345 diag("pthread_mutex_lock: %s",
346 strerror(errno));
347 abort();
b57aee66 348 }
ad460058
MD
349 ret = pthread_cond_signal(&call_rcu_test_cond);
350 if (ret) {
351 errno = ret;
352 diag("pthread_cond_signal: %s",
353 strerror(errno));
354 abort();
b57aee66 355 }
ad460058
MD
356 ret = pthread_mutex_unlock(&call_rcu_test_mutex);
357 if (ret) {
358 errno = ret;
359 diag("pthread_mutex_unlock: %s",
360 strerror(errno));
361 abort();
b57aee66
PM
362 }
363}
364
8a953620
MD
365void *rcu_update_stress_test(void *arg)
366{
367 int i;
368 struct rcu_stress *p;
b57aee66 369 struct rcu_head rh;
8a953620
MD
370
371 while (goflag == GOFLAG_INIT)
775aff2e 372 (void) poll(NULL, 0, 1);
8a953620
MD
373 while (goflag == GOFLAG_RUN) {
374 i = rcu_stress_idx + 1;
375 if (i >= RCU_STRESS_PIPE_LEN)
376 i = 0;
377 p = &rcu_stress_array[i];
378 p->mbtest = 0;
5481ddb3 379 cmm_smp_mb();
8a953620
MD
380 p->pipe_count = 0;
381 p->mbtest = 1;
382 rcu_assign_pointer(rcu_stress_current, p);
383 rcu_stress_idx = i;
384 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
385 if (i != rcu_stress_idx)
386 rcu_stress_array[i].pipe_count++;
b57aee66
PM
387 if (n_updates & 0x1)
388 synchronize_rcu();
389 else {
ad460058
MD
390 int ret;
391
392 ret = pthread_mutex_lock(&call_rcu_test_mutex);
393 if (ret) {
394 errno = ret;
395 diag("pthread_mutex_lock: %s",
396 strerror(errno));
397 abort();
b57aee66 398 }
0b9c513b 399 rcu_register_thread();
b57aee66 400 call_rcu(&rh, rcu_update_stress_test_rcu);
0b9c513b
MD
401 rcu_unregister_thread();
402 /*
403 * Our MacOS X test machine with the following
404 * config:
405 * 15.6.0 Darwin Kernel Version 15.6.0
406 * root:xnu-3248.60.10~1/RELEASE_X86_64
407 * appears to have issues with liburcu-signal
408 * signal being delivered on top of
409 * pthread_cond_wait. It seems to make the
410 * thread continue, and therefore corrupt the
411 * rcu_head. Work around this issue by
412 * unregistering the RCU read-side thread
413 * immediately after call_rcu (call_rcu needs
414 * us to be registered RCU readers).
415 */
ad460058
MD
416 ret = pthread_cond_wait(&call_rcu_test_cond,
417 &call_rcu_test_mutex);
418 if (ret) {
419 errno = ret;
420 diag("pthread_cond_signal: %s",
421 strerror(errno));
422 abort();
b57aee66 423 }
ad460058
MD
424 ret = pthread_mutex_unlock(&call_rcu_test_mutex);
425 if (ret) {
426 errno = ret;
427 diag("pthread_mutex_unlock: %s",
428 strerror(errno));
429 abort();
b57aee66
PM
430 }
431 }
8a953620
MD
432 n_updates++;
433 }
a13ef613 434
b0b31506 435 return NULL;
8a953620
MD
436}
437
438void *rcu_fake_update_stress_test(void *arg)
439{
26b5a74b 440 if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66
PM
441 struct call_rcu_data *crdp;
442
c1d2c60b 443 crdp = create_call_rcu_data(0, -1);
b57aee66 444 if (crdp != NULL) {
26b5a74b 445 diag("Successfully using per-thread call_rcu() worker.");
b57aee66
PM
446 set_thread_call_rcu_data(crdp);
447 }
448 }
8a953620 449 while (goflag == GOFLAG_INIT)
775aff2e 450 (void) poll(NULL, 0, 1);
8a953620
MD
451 while (goflag == GOFLAG_RUN) {
452 synchronize_rcu();
775aff2e 453 (void) poll(NULL, 0, 1);
8a953620 454 }
26b5a74b
MD
455 if (callrcu_type == CALLRCU_PERTHREAD) {
456 struct call_rcu_data *crdp;
457
458 crdp = get_thread_call_rcu_data();
459 set_thread_call_rcu_data(NULL);
460 call_rcu_data_free(crdp);
461 }
b0b31506 462 return NULL;
8a953620
MD
463}
464
ad460058 465int stresstest(int nreaders)
8a953620
MD
466{
467 int i;
468 int t;
469 long long *p;
470 long long sum;
471
472 init_per_thread(n_reads_pt, 0LL);
473 for_each_thread(t) {
474 p = &per_thread(rcu_stress_count,t)[0];
475 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
476 p[i] = 0LL;
477 }
478 rcu_stress_current = &rcu_stress_array[0];
479 rcu_stress_current->pipe_count = 0;
480 rcu_stress_current->mbtest = 1;
481 for (i = 0; i < nreaders; i++)
482 create_thread(rcu_read_stress_test, NULL);
483 create_thread(rcu_update_stress_test, NULL);
484 for (i = 0; i < 5; i++)
485 create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3 486 cmm_smp_mb();
8a953620 487 goflag = GOFLAG_RUN;
5481ddb3 488 cmm_smp_mb();
8a953620 489 sleep(10);
5481ddb3 490 cmm_smp_mb();
8a953620 491 goflag = GOFLAG_STOP;
5481ddb3 492 cmm_smp_mb();
8a953620
MD
493 wait_all_threads();
494 for_each_thread(t)
495 n_reads += per_thread(n_reads_pt, t);
ad460058 496 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
8a953620 497 n_reads, n_updates, n_mberror);
ad460058
MD
498 rdiag_start();
499 rdiag("rcu_stress_count:");
8a953620
MD
500 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
501 sum = 0LL;
502 for_each_thread(t) {
503 sum += per_thread(rcu_stress_count, t)[i];
504 }
ad460058 505 rdiag(" %lld", sum);
8a953620 506 }
ad460058 507 rdiag_end();
7106ddf8 508 if (get_cpu_call_rcu_data(0)) {
ad460058 509 diag("Deallocating per-CPU call_rcu threads.");
7106ddf8
PM
510 free_all_cpu_call_rcu_data();
511 }
ad460058
MD
512 if (!n_mberror)
513 return 0;
514 else
515 return -1;
8a953620
MD
516}
517
518/*
519 * Mainprogram.
520 */
521
522void usage(int argc, char *argv[])
523{
26b5a74b 524 diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
8a953620
MD
525 exit(-1);
526}
527
528int main(int argc, char *argv[])
529{
530 int nreaders = 1;
531 int cpustride = 1;
532
ad460058
MD
533 plan_tests(NR_TESTS);
534
8a953620
MD
535 smp_init();
536 //rcu_init();
26b5a74b
MD
537 if (argc > 4) {
538 const char *callrcu_str = argv[4];;
539
540 if (strcmp(callrcu_str, "callrcu_global") == 0) {
541 callrcu_type = CALLRCU_GLOBAL;
542 } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
543 callrcu_type = CALLRCU_PERCPU;
544 } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
545 callrcu_type = CALLRCU_PERTHREAD;
546 } else {
547 usage(argc, argv);
548 goto end;
549 }
550 }
929cfaff 551
26b5a74b
MD
552 switch (callrcu_type) {
553 case CALLRCU_GLOBAL:
554 diag("Using global per-process call_rcu thread.");
555 break;
556 case CALLRCU_PERCPU:
557 diag("Using per-CPU call_rcu threads.");
b57aee66 558 if (create_all_cpu_call_rcu_data(0))
ad460058
MD
559 diag("create_all_cpu_call_rcu_data: %s",
560 strerror(errno));
26b5a74b
MD
561 break;
562 case CALLRCU_PERTHREAD:
563 diag("Using per-thread call_rcu() worker.");
564 break;
565 default:
566 abort();
b57aee66 567 }
8a953620 568
9b171f46
MD
569#ifdef DEBUG_YIELD
570 yield_active |= YIELD_READ;
571 yield_active |= YIELD_WRITE;
572#endif
573
8a953620 574 if (argc > 1) {
26b5a74b
MD
575 if (strcmp(argv[1], "-h") == 0
576 || strcmp(argv[1], "--help") == 0) {
577 usage(argc, argv);
578 goto end;
579 }
8a953620 580 nreaders = strtoul(argv[1], NULL, 0);
ad460058
MD
581 if (argc == 2) {
582 ok(!perftest(nreaders, cpustride),
583 "perftest readers: %d, stride: %d",
584 nreaders, cpustride);
585 goto end;
586 }
8a953620
MD
587 if (argc > 3)
588 cpustride = strtoul(argv[3], NULL, 0);
589 if (strcmp(argv[2], "perf") == 0)
ad460058
MD
590 ok(!perftest(nreaders, cpustride),
591 "perftest readers: %d, stride: %d",
592 nreaders, cpustride);
8a953620 593 else if (strcmp(argv[2], "rperf") == 0)
ad460058
MD
594 ok(!rperftest(nreaders, cpustride),
595 "rperftest readers: %d, stride: %d",
596 nreaders, cpustride);
8a953620 597 else if (strcmp(argv[2], "uperf") == 0)
ad460058
MD
598 ok(!uperftest(nreaders, cpustride),
599 "uperftest readers: %d, stride: %d",
600 nreaders, cpustride);
8a953620 601 else if (strcmp(argv[2], "stress") == 0)
ad460058
MD
602 ok(!stresstest(nreaders),
603 "stresstest readers: %d, stride: %d",
604 nreaders, cpustride);
605 else
606 usage(argc, argv);
607 } else {
26b5a74b 608 usage(argc, argv);
8a953620 609 }
ad460058
MD
610end:
611 return exit_status();
8a953620 612}
This page took 0.064887 seconds and 4 git commands to generate.