urcu-wait: Fix wait state load/store
[urcu.git] / tests / regression / rcutorture.h
1 // SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation.
2 //
3 // SPDX-License-Identifier: GPL-2.0-or-later
4
5 /*
6 * rcutorture.h: simple user-level performance/stress test of RCU.
7 *
8 * Usage:
9 * ./rcu <nreaders> rperf [ <cpustride> ]
10 * Run a read-side performance test with the specified
11 * number of readers spaced by <cpustride>.
12 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
13 * CPUs from 0 to 30.
14 * ./rcu <nupdaters> uperf [ <cpustride> ]
15 * Run an update-side performance test with the specified
16 * number of updaters and specified CPU spacing.
17 * ./rcu <nreaders> perf [ <cpustride> ]
18 * Run a combined read/update performance test with the specified
19 * number of readers and one updater and specified CPU spacing.
20 * The readers run on the low-numbered CPUs and the updater
21 * of the highest-numbered CPU.
22 *
23 * The above tests produce output as follows:
24 *
25 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
26 * ns/read: 43.4707 ns/update: 6848.1
27 *
28 * The first line lists the total number of RCU reads and updates executed
29 * during the test, the number of reader threads, the number of updater
30 * threads, and the duration of the test in seconds. The second line
31 * lists the average duration of each type of operation in nanoseconds,
32 * or "nan" if the corresponding type of operation was not performed.
33 *
34 * ./rcu <nreaders> stress
35 * Run a stress test with the specified number of readers and
36 * one updater. None of the threads are affinitied to any
37 * particular CPU.
38 *
39 * This test produces output as follows:
40 *
41 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
42 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
43 *
44 * The first line lists the number of RCU read and update operations
45 * executed, followed by the number of memory-ordering violations
46 * (which will be zero in a correct RCU implementation). The second
47 * line lists the number of readers observing progressively more stale
48 * data. A correct RCU implementation will have all but the first two
49 * numbers non-zero.
50 */
51
52 /*
53 * Test variables.
54 */
55
56 #include <stdlib.h>
57 #include "tap.h"
58
59 #include "urcu-wait.h"
60
61 #define NR_TESTS 1
62
63 DEFINE_PER_THREAD(long long, n_reads_pt);
64 DEFINE_PER_THREAD(long long, n_updates_pt);
65
66 enum callrcu_type {
67 CALLRCU_GLOBAL,
68 CALLRCU_PERCPU,
69 CALLRCU_PERTHREAD,
70 };
71
72 enum writer_state {
73 WRITER_STATE_SYNC_RCU,
74 WRITER_STATE_CALL_RCU,
75 WRITER_STATE_POLL_RCU,
76 };
77
78 static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
79
80 long long n_reads = 0LL;
81 long n_updates = 0L;
82 int nthreadsrunning;
83 char argsbuf[64];
84
85 #define GOFLAG_INIT 0
86 #define GOFLAG_RUN 1
87 #define GOFLAG_STOP 2
88
89 volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
90 = GOFLAG_INIT;
91
92 #define RCU_READ_RUN 1000
93
94 //MD
95 #define RCU_READ_NESTABLE
96
97 #ifdef RCU_READ_NESTABLE
98 #define rcu_read_lock_nest() rcu_read_lock()
99 #define rcu_read_unlock_nest() rcu_read_unlock()
100 #else /* #ifdef RCU_READ_NESTABLE */
101 #define rcu_read_lock_nest()
102 #define rcu_read_unlock_nest()
103 #endif /* #else #ifdef RCU_READ_NESTABLE */
104
105 #ifdef TORTURE_QSBR
106 #define mark_rcu_quiescent_state rcu_quiescent_state
107 #define put_thread_offline rcu_thread_offline
108 #define put_thread_online rcu_thread_online
109 #endif
110
111 #ifndef mark_rcu_quiescent_state
112 #define mark_rcu_quiescent_state() do {} while (0)
113 #endif /* #ifdef mark_rcu_quiescent_state */
114
115 #ifndef put_thread_offline
116 #define put_thread_offline() do {} while (0)
117 #define put_thread_online() do {} while (0)
118 #define put_thread_online_delay() do {} while (0)
119 #else /* #ifndef put_thread_offline */
120 #define put_thread_online_delay() synchronize_rcu()
121 #endif /* #else #ifndef put_thread_offline */
122
123 /*
124 * Performance test.
125 */
126
127 static
128 void *rcu_read_perf_test(void *arg)
129 {
130 int i;
131 int me = (long)arg;
132 long long n_reads_local = 0;
133
134 rcu_register_thread();
135 run_on(me);
136 uatomic_inc(&nthreadsrunning);
137 put_thread_offline();
138 while (goflag == GOFLAG_INIT)
139 (void) poll(NULL, 0, 1);
140 put_thread_online();
141 while (goflag == GOFLAG_RUN) {
142 for (i = 0; i < RCU_READ_RUN; i++) {
143 rcu_read_lock();
144 /* rcu_read_lock_nest(); */
145 /* rcu_read_unlock_nest(); */
146 rcu_read_unlock();
147 }
148 n_reads_local += RCU_READ_RUN;
149 mark_rcu_quiescent_state();
150 }
151 __get_thread_var(n_reads_pt) += n_reads_local;
152 put_thread_offline();
153 rcu_unregister_thread();
154
155 return (NULL);
156 }
157
158 static
159 void *rcu_update_perf_test(void *arg __attribute__((unused)))
160 {
161 long long n_updates_local = 0;
162
163 if (callrcu_type == CALLRCU_PERTHREAD) {
164 struct call_rcu_data *crdp;
165
166 crdp = create_call_rcu_data(0, -1);
167 if (crdp != NULL) {
168 diag("Successfully using per-thread call_rcu() worker.");
169 set_thread_call_rcu_data(crdp);
170 }
171 }
172 uatomic_inc(&nthreadsrunning);
173 while (goflag == GOFLAG_INIT)
174 (void) poll(NULL, 0, 1);
175 while (goflag == GOFLAG_RUN) {
176 synchronize_rcu();
177 n_updates_local++;
178 }
179 __get_thread_var(n_updates_pt) += n_updates_local;
180 if (callrcu_type == CALLRCU_PERTHREAD) {
181 struct call_rcu_data *crdp;
182
183 crdp = get_thread_call_rcu_data();
184 set_thread_call_rcu_data(NULL);
185 call_rcu_data_free(crdp);
186 }
187 return NULL;
188 }
189
190 static
191 void perftestinit(void)
192 {
193 init_per_thread(n_reads_pt, 0LL);
194 init_per_thread(n_updates_pt, 0LL);
195 uatomic_set(&nthreadsrunning, 0);
196 }
197
198 static
199 int perftestrun(int nthreads, int nreaders, int nupdaters)
200 {
201 int t;
202 int duration = 1;
203
204 cmm_smp_mb();
205 while (uatomic_read(&nthreadsrunning) < nthreads)
206 (void) poll(NULL, 0, 1);
207 goflag = GOFLAG_RUN;
208 cmm_smp_mb();
209 sleep(duration);
210 cmm_smp_mb();
211 goflag = GOFLAG_STOP;
212 cmm_smp_mb();
213 wait_all_threads();
214 for_each_thread(t) {
215 n_reads += per_thread(n_reads_pt, t);
216 n_updates += per_thread(n_updates_pt, t);
217 }
218 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
219 n_reads, n_updates, nreaders, nupdaters, duration);
220 diag("ns/read: %g ns/update: %g",
221 ((duration * 1000*1000*1000.*(double)nreaders) /
222 (double)n_reads),
223 ((duration * 1000*1000*1000.*(double)nupdaters) /
224 (double)n_updates));
225 if (get_cpu_call_rcu_data(0)) {
226 diag("Deallocating per-CPU call_rcu threads.\n");
227 free_all_cpu_call_rcu_data();
228 }
229 return 0;
230 }
231
232 static
233 int perftest(int nreaders, int cpustride)
234 {
235 int i;
236 long arg;
237
238 perftestinit();
239 for (i = 0; i < nreaders; i++) {
240 arg = (long)(i * cpustride);
241 create_thread(rcu_read_perf_test, (void *)arg);
242 }
243 arg = (long)(i * cpustride);
244 create_thread(rcu_update_perf_test, (void *)arg);
245 return perftestrun(i + 1, nreaders, 1);
246 }
247
248 static
249 int rperftest(int nreaders, int cpustride)
250 {
251 int i;
252 long arg;
253
254 perftestinit();
255 init_per_thread(n_reads_pt, 0LL);
256 for (i = 0; i < nreaders; i++) {
257 arg = (long)(i * cpustride);
258 create_thread(rcu_read_perf_test, (void *)arg);
259 }
260 return perftestrun(i, nreaders, 0);
261 }
262
263 static
264 int uperftest(int nupdaters, int cpustride)
265 {
266 int i;
267 long arg;
268
269 perftestinit();
270 init_per_thread(n_reads_pt, 0LL);
271 for (i = 0; i < nupdaters; i++) {
272 arg = (long)(i * cpustride);
273 create_thread(rcu_update_perf_test, (void *)arg);
274 }
275 return perftestrun(i, 0, nupdaters);
276 }
277
278 /*
279 * Stress test.
280 */
281
282 #define RCU_STRESS_PIPE_LEN 10
283
284 struct rcu_stress {
285 int pipe_count;
286 int mbtest;
287 };
288
289 struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0, 0 } };
290 struct rcu_stress *rcu_stress_current;
291 int rcu_stress_idx = 0;
292
293 int n_mberror = 0;
294 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
295
296 int garbage = 0;
297
298 static
299 void *rcu_read_stress_test(void *arg __attribute__((unused)))
300 {
301 int i;
302 int itercnt = 0;
303 struct rcu_stress *p;
304 int pc;
305
306 rcu_register_thread();
307 put_thread_offline();
308 while (goflag == GOFLAG_INIT)
309 (void) poll(NULL, 0, 1);
310 put_thread_online();
311 while (goflag == GOFLAG_RUN) {
312 rcu_read_lock();
313 p = rcu_dereference(rcu_stress_current);
314 if (p->mbtest == 0)
315 n_mberror++;
316 rcu_read_lock_nest();
317 for (i = 0; i < 100; i++)
318 garbage++;
319 rcu_read_unlock_nest();
320 pc = p->pipe_count;
321 rcu_read_unlock();
322 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
323 pc = RCU_STRESS_PIPE_LEN;
324 __get_thread_var(rcu_stress_count)[pc]++;
325 __get_thread_var(n_reads_pt)++;
326 mark_rcu_quiescent_state();
327 if ((++itercnt % 0x1000) == 0) {
328 put_thread_offline();
329 put_thread_online_delay();
330 put_thread_online();
331 }
332 }
333 put_thread_offline();
334 rcu_unregister_thread();
335
336 return (NULL);
337 }
338
339 static DEFINE_URCU_WAIT_QUEUE(call_rcu_waiters);
340
341 static
342 void rcu_update_stress_test_rcu(struct rcu_head *head __attribute__((unused)))
343 {
344 struct urcu_waiters waiters;
345
346 urcu_move_waiters(&waiters, &call_rcu_waiters);
347 urcu_wake_all_waiters(&waiters);
348 }
349
350 static
351 void advance_writer_state(enum writer_state *state)
352 {
353 switch (*state) {
354 case WRITER_STATE_SYNC_RCU:
355 *state = WRITER_STATE_CALL_RCU;
356 break;
357 case WRITER_STATE_CALL_RCU:
358 *state = WRITER_STATE_POLL_RCU;
359 break;
360 case WRITER_STATE_POLL_RCU:
361 *state = WRITER_STATE_SYNC_RCU;
362 break;
363 }
364 }
365
366 static
367 void *rcu_update_stress_test(void *arg __attribute__((unused)))
368 {
369 int i;
370 struct rcu_stress *p;
371 struct rcu_head rh;
372 enum writer_state writer_state = WRITER_STATE_SYNC_RCU;
373
374 rcu_register_thread();
375
376 /* Offline for poll. */
377 put_thread_offline();
378 while (goflag == GOFLAG_INIT)
379 (void) poll(NULL, 0, 1);
380 put_thread_online();
381
382 while (goflag == GOFLAG_RUN) {
383 i = rcu_stress_idx + 1;
384 if (i >= RCU_STRESS_PIPE_LEN)
385 i = 0;
386 p = &rcu_stress_array[i];
387 p->mbtest = 0;
388 cmm_smp_mb();
389 p->pipe_count = 0;
390 p->mbtest = 1;
391 rcu_assign_pointer(rcu_stress_current, p);
392 rcu_stress_idx = i;
393 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
394 if (i != rcu_stress_idx)
395 rcu_stress_array[i].pipe_count++;
396 switch (writer_state) {
397 case WRITER_STATE_SYNC_RCU:
398 synchronize_rcu();
399 break;
400 case WRITER_STATE_CALL_RCU:
401 {
402 DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
403
404 urcu_wait_add(&call_rcu_waiters, &wait);
405
406 call_rcu(&rh, rcu_update_stress_test_rcu);
407
408 /* Offline for busy-wait. */
409 put_thread_offline();
410 urcu_adaptative_busy_wait(&wait);
411 put_thread_online();
412 break;
413 }
414 case WRITER_STATE_POLL_RCU:
415 {
416 struct urcu_gp_poll_state poll_state;
417
418 poll_state = start_poll_synchronize_rcu();
419
420 /* Offline for poll. */
421 put_thread_offline();
422 while (!poll_state_synchronize_rcu(poll_state))
423 (void) poll(NULL, 0, 1); /* Wait for 1ms */
424 put_thread_online();
425 break;
426 }
427 }
428 n_updates++;
429 advance_writer_state(&writer_state);
430 }
431
432 rcu_unregister_thread();
433
434 return NULL;
435 }
436
437 static
438 void *rcu_fake_update_stress_test(void *arg __attribute__((unused)))
439 {
440 if (callrcu_type == CALLRCU_PERTHREAD) {
441 struct call_rcu_data *crdp;
442
443 crdp = create_call_rcu_data(0, -1);
444 if (crdp != NULL) {
445 diag("Successfully using per-thread call_rcu() worker.");
446 set_thread_call_rcu_data(crdp);
447 }
448 }
449 while (goflag == GOFLAG_INIT)
450 (void) poll(NULL, 0, 1);
451 while (goflag == GOFLAG_RUN) {
452 synchronize_rcu();
453 (void) poll(NULL, 0, 1);
454 }
455 if (callrcu_type == CALLRCU_PERTHREAD) {
456 struct call_rcu_data *crdp;
457
458 crdp = get_thread_call_rcu_data();
459 set_thread_call_rcu_data(NULL);
460 call_rcu_data_free(crdp);
461 }
462 return NULL;
463 }
464
465 static
466 int stresstest(int nreaders)
467 {
468 int i;
469 int t;
470 long long *p;
471 long long sum;
472
473 init_per_thread(n_reads_pt, 0LL);
474 for_each_thread(t) {
475 p = &per_thread(rcu_stress_count,t)[0];
476 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
477 p[i] = 0LL;
478 }
479 rcu_stress_current = &rcu_stress_array[0];
480 rcu_stress_current->pipe_count = 0;
481 rcu_stress_current->mbtest = 1;
482 for (i = 0; i < nreaders; i++)
483 create_thread(rcu_read_stress_test, NULL);
484 create_thread(rcu_update_stress_test, NULL);
485 for (i = 0; i < 5; i++)
486 create_thread(rcu_fake_update_stress_test, NULL);
487 cmm_smp_mb();
488 goflag = GOFLAG_RUN;
489 cmm_smp_mb();
490 sleep(10);
491 cmm_smp_mb();
492 goflag = GOFLAG_STOP;
493 cmm_smp_mb();
494 wait_all_threads();
495 for_each_thread(t)
496 n_reads += per_thread(n_reads_pt, t);
497 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
498 n_reads, n_updates, n_mberror);
499 rdiag_start();
500 rdiag("rcu_stress_count:");
501 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
502 sum = 0LL;
503 for_each_thread(t) {
504 sum += per_thread(rcu_stress_count, t)[i];
505 }
506 rdiag(" %lld", sum);
507 }
508 rdiag_end();
509 if (get_cpu_call_rcu_data(0)) {
510 diag("Deallocating per-CPU call_rcu threads.");
511 free_all_cpu_call_rcu_data();
512 }
513 if (!n_mberror)
514 return 0;
515 else
516 return -1;
517 }
518
519 /*
520 * Mainprogram.
521 */
522
523 static
524 void usage(char *argv[]) __attribute__((__noreturn__));
525
526 static
527 void usage(char *argv[])
528 {
529 diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
530 exit(-1);
531 }
532
533 int main(int argc, char *argv[])
534 {
535 int nreaders = 1;
536 int cpustride = 1;
537
538 plan_tests(NR_TESTS);
539
540 smp_init();
541 //rcu_init();
542 if (argc > 4) {
543 const char *callrcu_str = argv[4];;
544
545 if (strcmp(callrcu_str, "callrcu_global") == 0) {
546 callrcu_type = CALLRCU_GLOBAL;
547 } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
548 callrcu_type = CALLRCU_PERCPU;
549 } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
550 callrcu_type = CALLRCU_PERTHREAD;
551 } else {
552 usage(argv);
553 goto end;
554 }
555 }
556
557 switch (callrcu_type) {
558 case CALLRCU_GLOBAL:
559 diag("Using global per-process call_rcu thread.");
560 break;
561 case CALLRCU_PERCPU:
562 diag("Using per-CPU call_rcu threads.");
563 if (create_all_cpu_call_rcu_data(0))
564 diag("create_all_cpu_call_rcu_data: %s",
565 strerror(errno));
566 break;
567 case CALLRCU_PERTHREAD:
568 diag("Using per-thread call_rcu() worker.");
569 break;
570 default:
571 abort();
572 }
573
574 #ifdef DEBUG_YIELD
575 yield_active |= YIELD_READ;
576 yield_active |= YIELD_WRITE;
577 #endif
578
579 if (argc > 1) {
580 if (strcmp(argv[1], "-h") == 0
581 || strcmp(argv[1], "--help") == 0) {
582 usage(argv);
583 goto end;
584 }
585 nreaders = strtoul(argv[1], NULL, 0);
586 if (argc == 2) {
587 ok(!perftest(nreaders, cpustride),
588 "perftest readers: %d, stride: %d",
589 nreaders, cpustride);
590 goto end;
591 }
592 if (argc > 3)
593 cpustride = strtoul(argv[3], NULL, 0);
594 if (strcmp(argv[2], "perf") == 0)
595 ok(!perftest(nreaders, cpustride),
596 "perftest readers: %d, stride: %d",
597 nreaders, cpustride);
598 else if (strcmp(argv[2], "rperf") == 0)
599 ok(!rperftest(nreaders, cpustride),
600 "rperftest readers: %d, stride: %d",
601 nreaders, cpustride);
602 else if (strcmp(argv[2], "uperf") == 0)
603 ok(!uperftest(nreaders, cpustride),
604 "uperftest readers: %d, stride: %d",
605 nreaders, cpustride);
606 else if (strcmp(argv[2], "stress") == 0)
607 ok(!stresstest(nreaders),
608 "stresstest readers: %d, stride: %d",
609 nreaders, cpustride);
610 else
611 usage(argv);
612 } else {
613 usage(argv);
614 }
615 end:
616 return exit_status();
617 }
This page took 0.044133 seconds and 4 git commands to generate.