[urcu.git] / tests / regression / rcutorture.h

/*
 * rcutorture.h: simple user-level performance/stress test of RCU.
 *
 * Usage:
 * 	./rcu <nreaders> rperf [ <cpustride> ]
 * 		Run a read-side performance test with the specified
 * 		number of readers spaced by <cpustride>.
 * 		Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
 * 		CPUs from 0 to 30.
 * 	./rcu <nupdaters> uperf [ <cpustride> ]
 * 		Run an update-side performance test with the specified
 * 		number of updaters and specified CPU spacing.
 * 	./rcu <nreaders> perf [ <cpustride> ]
 * 		Run a combined read/update performance test with the specified
 * 		number of readers and one updater and specified CPU spacing.
 * 		The readers run on the low-numbered CPUs and the updater
 * 		of the highest-numbered CPU.
 *
 * The above tests produce output as follows:
 *
 * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
 * ns/read: 43.4707  ns/update: 6848.1
 *
 * The first line lists the total number of RCU reads and updates executed
 * during the test, the number of reader threads, the number of updater
 * threads, and the duration of the test in seconds.  The second line
 * lists the average duration of each type of operation in nanoseconds,
 * or "nan" if the corresponding type of operation was not performed.
 *
 * 	./rcu <nreaders> stress
 * 		Run a stress test with the specified number of readers and
 * 		one updater.  None of the threads are affinitied to any
 * 		particular CPU.
 *
 * This test produces output as follows:
 *
 * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
 *
 * The first line lists the number of RCU read and update operations
 * executed, followed by the number of memory-ordering violations
 * (which will be zero in a correct RCU implementation).  The second
 * line lists the number of readers observing progressively more stale
 * data.  A correct RCU implementation will have all but the first two
 * numbers non-zero.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
 */

/*
 * Test variables.
 */

#include <stdlib.h>

DEFINE_PER_THREAD(long long, n_reads_pt);
DEFINE_PER_THREAD(long long, n_updates_pt);

long long n_reads = 0LL;
long n_updates = 0L;
int nthreadsrunning;
char argsbuf[64];

#define GOFLAG_INIT 0
#define GOFLAG_RUN  1
#define GOFLAG_STOP 2

volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
        = GOFLAG_INIT;

#define RCU_READ_RUN 1000

//MD
#define RCU_READ_NESTABLE

#ifdef RCU_READ_NESTABLE
#define rcu_read_lock_nest() rcu_read_lock()
#define rcu_read_unlock_nest() rcu_read_unlock()
#else /* #ifdef RCU_READ_NESTABLE */
#define rcu_read_lock_nest()
#define rcu_read_unlock_nest()
#endif /* #else #ifdef RCU_READ_NESTABLE */

#ifdef TORTURE_QSBR
#define mark_rcu_quiescent_state	rcu_quiescent_state
#define put_thread_offline		rcu_thread_offline
#define put_thread_online		rcu_thread_online
#endif

#ifndef mark_rcu_quiescent_state
#define mark_rcu_quiescent_state() do ; while (0)
#endif /* #ifdef mark_rcu_quiescent_state */

#ifndef put_thread_offline
#define put_thread_offline()		do ; while (0)
#define put_thread_online()		do ; while (0)
#define put_thread_online_delay()	do ; while (0)
#else /* #ifndef put_thread_offline */
#define put_thread_online_delay()	synchronize_rcu()
#endif /* #else #ifndef put_thread_offline */

/*
 * Performance test.
 */

void *rcu_read_perf_test(void *arg)
{
	struct call_rcu_data *crdp;
	int i;
	int me = (long)arg;
	long long n_reads_local = 0;

	rcu_register_thread();
	run_on(me);
	uatomic_inc(&nthreadsrunning);
	put_thread_offline();
	while (goflag == GOFLAG_INIT)
		poll(NULL, 0, 1);
	put_thread_online();
	while (goflag == GOFLAG_RUN) {
		for (i = 0; i < RCU_READ_RUN; i++) {
			rcu_read_lock();
			/* rcu_read_lock_nest(); */
			/* rcu_read_unlock_nest(); */
			rcu_read_unlock();
		}
		n_reads_local += RCU_READ_RUN;
		mark_rcu_quiescent_state();
	}
	__get_thread_var(n_reads_pt) += n_reads_local;
	put_thread_offline();
	crdp = get_thread_call_rcu_data();
	set_thread_call_rcu_data(NULL);
	call_rcu_data_free(crdp);
	rcu_unregister_thread();

	return (NULL);
}

void *rcu_update_perf_test(void *arg)
{
	long long n_updates_local = 0;

	if ((random() & 0xf00) == 0) {
		struct call_rcu_data *crdp;

		crdp = create_call_rcu_data(0, -1);
		if (crdp != NULL) {
			fprintf(stderr,
				"Using per-thread call_rcu() worker.\n");
			set_thread_call_rcu_data(crdp);
		}
	}
	uatomic_inc(&nthreadsrunning);
	while (goflag == GOFLAG_INIT)
		poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		synchronize_rcu();
		n_updates_local++;
	}
	__get_thread_var(n_updates_pt) += n_updates_local;
	return NULL;
}

void perftestinit(void)
{
	init_per_thread(n_reads_pt, 0LL);
	init_per_thread(n_updates_pt, 0LL);
	uatomic_set(&nthreadsrunning, 0);
}

void perftestrun(int nthreads, int nreaders, int nupdaters)
{
	int t;
	int duration = 1;

	cmm_smp_mb();
	while (uatomic_read(&nthreadsrunning) < nthreads)
		poll(NULL, 0, 1);
	goflag = GOFLAG_RUN;
	cmm_smp_mb();
	sleep(duration);
	cmm_smp_mb();
	goflag = GOFLAG_STOP;
	cmm_smp_mb();
	wait_all_threads();
	for_each_thread(t) {
		n_reads += per_thread(n_reads_pt, t);
		n_updates += per_thread(n_updates_pt, t);
	}
	printf("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d\n",
	       n_reads, n_updates, nreaders, nupdaters, duration);
	printf("ns/read: %g  ns/update: %g\n",
	       ((duration * 1000*1000*1000.*(double)nreaders) /
	        (double)n_reads),
	       ((duration * 1000*1000*1000.*(double)nupdaters) /
	        (double)n_updates));
	if (get_cpu_call_rcu_data(0)) {
		fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n");
		free_all_cpu_call_rcu_data();
	}
	exit(0);
}

void perftest(int nreaders, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	for (i = 0; i < nreaders; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_read_perf_test, (void *)arg);
	}
	arg = (long)(i * cpustride);
	create_thread(rcu_update_perf_test, (void *)arg);
	perftestrun(i + 1, nreaders, 1);
}

void rperftest(int nreaders, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	init_per_thread(n_reads_pt, 0LL);
	for (i = 0; i < nreaders; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_read_perf_test, (void *)arg);
	}
	perftestrun(i, nreaders, 0);
}

void uperftest(int nupdaters, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	init_per_thread(n_reads_pt, 0LL);
	for (i = 0; i < nupdaters; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_update_perf_test, (void *)arg);
	}
	perftestrun(i, 0, nupdaters);
}

/*
 * Stress test.
 */

#define RCU_STRESS_PIPE_LEN 10

struct rcu_stress {
	int pipe_count;
	int mbtest;
};

struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
struct rcu_stress *rcu_stress_current;
int rcu_stress_idx = 0;

int n_mberror = 0;
DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);

int garbage = 0;

void *rcu_read_stress_test(void *arg)
{
	int i;
	int itercnt = 0;
	struct rcu_stress *p;
	int pc;

	rcu_register_thread();
	put_thread_offline();
	while (goflag == GOFLAG_INIT)
		poll(NULL, 0, 1);
	put_thread_online();
	while (goflag == GOFLAG_RUN) {
		rcu_read_lock();
		p = rcu_dereference(rcu_stress_current);
		if (p->mbtest == 0)
			n_mberror++;
		rcu_read_lock_nest();
		for (i = 0; i < 100; i++)
			garbage++;
		rcu_read_unlock_nest();
		pc = p->pipe_count;
		rcu_read_unlock();
		if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
			pc = RCU_STRESS_PIPE_LEN;
		__get_thread_var(rcu_stress_count)[pc]++;
		__get_thread_var(n_reads_pt)++;
		mark_rcu_quiescent_state();
		if ((++itercnt % 0x1000) == 0) {
			put_thread_offline();
			put_thread_online_delay();
			put_thread_online();
		}
	}
	put_thread_offline();
	rcu_unregister_thread();

	return (NULL);
}

static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;

void rcu_update_stress_test_rcu(struct rcu_head *head)
{
	if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) {
		perror("pthread_mutex_lock");
		exit(-1);
	}
	if (pthread_cond_signal(&call_rcu_test_cond) != 0) {
		perror("pthread_cond_signal");
		exit(-1);
	}
	if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) {
		perror("pthread_mutex_unlock");
		exit(-1);
	}
}

void *rcu_update_stress_test(void *arg)
{
	int i;
	struct rcu_stress *p;
	struct rcu_head rh;

	while (goflag == GOFLAG_INIT)
		poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		i = rcu_stress_idx + 1;
		if (i >= RCU_STRESS_PIPE_LEN)
			i = 0;
		p = &rcu_stress_array[i];
		p->mbtest = 0;
		cmm_smp_mb();
		p->pipe_count = 0;
		p->mbtest = 1;
		rcu_assign_pointer(rcu_stress_current, p);
		rcu_stress_idx = i;
		for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
			if (i != rcu_stress_idx)
				rcu_stress_array[i].pipe_count++;
		if (n_updates & 0x1)
			synchronize_rcu();
		else {
			if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) {
				perror("pthread_mutex_lock");
				exit(-1);
			}
			rcu_register_thread();
			call_rcu(&rh, rcu_update_stress_test_rcu);
			rcu_unregister_thread();
			/*
			 * Our MacOS X test machine with the following
			 * config:
			 * 15.6.0 Darwin Kernel Version 15.6.0
			 * root:xnu-3248.60.10~1/RELEASE_X86_64
			 * appears to have issues with liburcu-signal
			 * signal being delivered on top of
			 * pthread_cond_wait. It seems to make the
			 * thread continue, and therefore corrupt the
			 * rcu_head. Work around this issue by
			 * unregistering the RCU read-side thread
			 * immediately after call_rcu (call_rcu needs
			 * us to be registered RCU readers).
			 */
			if (pthread_cond_wait(&call_rcu_test_cond,
					      &call_rcu_test_mutex) != 0) {
				perror("pthread_cond_wait");
				exit(-1);
			}
			if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) {
				perror("pthread_mutex_unlock");
				exit(-1);
			}
		}
		n_updates++;
	}

	return NULL;
}

void *rcu_fake_update_stress_test(void *arg)
{
	if ((random() & 0xf00) == 0) {
		struct call_rcu_data *crdp;

		crdp = create_call_rcu_data(0, -1);
		if (crdp != NULL) {
			fprintf(stderr,
				"Using per-thread call_rcu() worker.\n");
			set_thread_call_rcu_data(crdp);
		}
	}
	while (goflag == GOFLAG_INIT)
		poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		synchronize_rcu();
		poll(NULL, 0, 1);
	}
	return NULL;
}

void stresstest(int nreaders)
{
	int i;
	int t;
	long long *p;
	long long sum;

	init_per_thread(n_reads_pt, 0LL);
	for_each_thread(t) {
		p = &per_thread(rcu_stress_count,t)[0];
		for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
			p[i] = 0LL;
	}
	rcu_stress_current = &rcu_stress_array[0];
	rcu_stress_current->pipe_count = 0;
	rcu_stress_current->mbtest = 1;
	for (i = 0; i < nreaders; i++)
		create_thread(rcu_read_stress_test, NULL);
	create_thread(rcu_update_stress_test, NULL);
	for (i = 0; i < 5; i++)
		create_thread(rcu_fake_update_stress_test, NULL);
	cmm_smp_mb();
	goflag = GOFLAG_RUN;
	cmm_smp_mb();
	sleep(10);
	cmm_smp_mb();
	goflag = GOFLAG_STOP;
	cmm_smp_mb();
	wait_all_threads();
	for_each_thread(t)
		n_reads += per_thread(n_reads_pt, t);
	printf("n_reads: %lld  n_updates: %ld  n_mberror: %d\n",
	       n_reads, n_updates, n_mberror);
	printf("rcu_stress_count:");
	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
		sum = 0LL;
		for_each_thread(t) {
			sum += per_thread(rcu_stress_count, t)[i];
		}
		printf(" %lld", sum);
	}
	printf("\n");
	if (get_cpu_call_rcu_data(0)) {
		fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n");
		free_all_cpu_call_rcu_data();
	}
	exit(0);
}

/*
 * Mainprogram.
 */

void usage(int argc, char *argv[])
{
	fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]);
	exit(-1);
}

int main(int argc, char *argv[])
{
	int nreaders = 1;
	int cpustride = 1;

	smp_init();
	//rcu_init();
	srandom(time(NULL));
	if (random() & 0x100) {
		fprintf(stderr, "Allocating per-CPU call_rcu threads.\n");
		if (create_all_cpu_call_rcu_data(0))
			perror("create_all_cpu_call_rcu_data");
	}

#ifdef DEBUG_YIELD
	yield_active |= YIELD_READ;
	yield_active |= YIELD_WRITE;
#endif

	if (argc > 1) {
		nreaders = strtoul(argv[1], NULL, 0);
		if (argc == 2)
			perftest(nreaders, cpustride);
		if (argc > 3)
			cpustride = strtoul(argv[3], NULL, 0);
		if (strcmp(argv[2], "perf") == 0)
			perftest(nreaders, cpustride);
		else if (strcmp(argv[2], "rperf") == 0)
			rperftest(nreaders, cpustride);
		else if (strcmp(argv[2], "uperf") == 0)
			uperftest(nreaders, cpustride);
		else if (strcmp(argv[2], "stress") == 0)
			stresstest(nreaders);
		usage(argc, argv);
	}
	perftest(nreaders, cpustride);
	return 0;
}
Commit	Line	Data
8a953620 MD	1	/*
	2	* rcutorture.h: simple user-level performance/stress test of RCU.
	3	*
	4	* Usage:
	5	* ./rcu <nreaders> rperf [ <cpustride> ]
	6	* Run a read-side performance test with the specified
	7	* number of readers spaced by <cpustride>.
	8	* Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
	9	* CPUs from 0 to 30.
	10	* ./rcu <nupdaters> uperf [ <cpustride> ]
	11	* Run an update-side performance test with the specified
	12	* number of updaters and specified CPU spacing.
	13	* ./rcu <nreaders> perf [ <cpustride> ]
	14	* Run a combined read/update performance test with the specified
	15	* number of readers and one updater and specified CPU spacing.
	16	* The readers run on the low-numbered CPUs and the updater
	17	* of the highest-numbered CPU.
	18	*
	19	* The above tests produce output as follows:
	20	*
	21	* n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
	22	* ns/read: 43.4707 ns/update: 6848.1
	23	*
	24	* The first line lists the total number of RCU reads and updates executed
	25	* during the test, the number of reader threads, the number of updater
	26	* threads, and the duration of the test in seconds. The second line
	27	* lists the average duration of each type of operation in nanoseconds,
	28	* or "nan" if the corresponding type of operation was not performed.
	29	*
	30	* ./rcu <nreaders> stress
	31	* Run a stress test with the specified number of readers and
	32	* one updater. None of the threads are affinitied to any
	33	* particular CPU.
	34	*
	35	* This test produces output as follows:
	36	*
	37	* n_reads: 114633217 n_updates: 3903415 n_mberror: 0
	38	* rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
	39	*
	40	* The first line lists the number of RCU read and update operations
	41	* executed, followed by the number of memory-ordering violations
	42	* (which will be zero in a correct RCU implementation). The second
	43	* line lists the number of readers observing progressively more stale
	44	* data. A correct RCU implementation will have all but the first two
	45	* numbers non-zero.
	46	*
	47	* This program is free software; you can redistribute it and/or modify
	48	* it under the terms of the GNU General Public License as published by
	49	* the Free Software Foundation; either version 2 of the License, or
	50	* (at your option) any later version.
	51	*
	52	* This program is distributed in the hope that it will be useful,
	53	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	54	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	55	* GNU General Public License for more details.
	56	*
	57	* You should have received a copy of the GNU General Public License
	58	* along with this program; if not, write to the Free Software
3282a76b	59	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
8a953620 MD	60	*
	61	* Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
	62	*/
	63
	64	/*
	65	* Test variables.
	66	*/
	67
b57aee66	68	#include <stdlib.h>
b57aee66	69
8a953620 MD	70	DEFINE_PER_THREAD(long long, n_reads_pt);
	71	DEFINE_PER_THREAD(long long, n_updates_pt);
	72
	73	long long n_reads = 0LL;
	74	long n_updates = 0L;
6ee91d83	75	int nthreadsrunning;
8a953620 MD	76	char argsbuf[64];
	77
	78	#define GOFLAG_INIT 0
	79	#define GOFLAG_RUN 1
	80	#define GOFLAG_STOP 2
	81
4967f005 PB	82	volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
4967f005 PB	83	= GOFLAG_INIT;
8a953620 MD	84
	85	#define RCU_READ_RUN 1000
	86
	87	//MD
	88	#define RCU_READ_NESTABLE
	89
	90	#ifdef RCU_READ_NESTABLE
	91	#define rcu_read_lock_nest() rcu_read_lock()
	92	#define rcu_read_unlock_nest() rcu_read_unlock()
	93	#else /* #ifdef RCU_READ_NESTABLE */
	94	#define rcu_read_lock_nest()
	95	#define rcu_read_unlock_nest()
	96	#endif /* #else #ifdef RCU_READ_NESTABLE */
	97
1a43bbd8 MD	98	#ifdef TORTURE_QSBR
	99	#define mark_rcu_quiescent_state rcu_quiescent_state
	100	#define put_thread_offline rcu_thread_offline
	101	#define put_thread_online rcu_thread_online
	102	#endif
	103
8a953620 MD	104	#ifndef mark_rcu_quiescent_state
	105	#define mark_rcu_quiescent_state() do ; while (0)
	106	#endif /* #ifdef mark_rcu_quiescent_state */
	107
	108	#ifndef put_thread_offline
	109	#define put_thread_offline() do ; while (0)
	110	#define put_thread_online() do ; while (0)
	111	#define put_thread_online_delay() do ; while (0)
	112	#else /* #ifndef put_thread_offline */
	113	#define put_thread_online_delay() synchronize_rcu()
	114	#endif /* #else #ifndef put_thread_offline */
	115
	116	/*
	117	* Performance test.
	118	*/
	119
	120	void rcu_read_perf_test(void arg)
	121	{
7106ddf8	122	struct call_rcu_data *crdp;
8a953620 MD	123	int i;
8a953620 MD	124	int me = (long)arg;
8a953620 MD	125	long long n_reads_local = 0;
8a953620 MD	126
121a5d44	127	rcu_register_thread();
8a953620	128	run_on(me);
ec4e58a3	129	uatomic_inc(&nthreadsrunning);
3bf02b5b	130	put_thread_offline();
8a953620 MD	131	while (goflag == GOFLAG_INIT)
8a953620 MD	132	poll(NULL, 0, 1);
3bf02b5b	133	put_thread_online();
8a953620 MD	134	while (goflag == GOFLAG_RUN) {
	135	for (i = 0; i < RCU_READ_RUN; i++) {
	136	rcu_read_lock();
	137	/* rcu_read_lock_nest(); */
	138	/* rcu_read_unlock_nest(); */
	139	rcu_read_unlock();
	140	}
	141	n_reads_local += RCU_READ_RUN;
	142	mark_rcu_quiescent_state();
	143	}
	144	__get_thread_var(n_reads_pt) += n_reads_local;
	145	put_thread_offline();
7106ddf8 PM	146	crdp = get_thread_call_rcu_data();
	147	set_thread_call_rcu_data(NULL);
	148	call_rcu_data_free(crdp);
121a5d44	149	rcu_unregister_thread();
8a953620 MD	150
	151	return (NULL);
	152	}
	153
	154	void rcu_update_perf_test(void arg)
	155	{
	156	long long n_updates_local = 0;
	157
b57aee66 PM	158	if ((random() & 0xf00) == 0) {
	159	struct call_rcu_data *crdp;
	160
c1d2c60b	161	crdp = create_call_rcu_data(0, -1);
b57aee66 PM	162	if (crdp != NULL) {
	163	fprintf(stderr,
	164	"Using per-thread call_rcu() worker.\n");
	165	set_thread_call_rcu_data(crdp);
	166	}
	167	}
ec4e58a3	168	uatomic_inc(&nthreadsrunning);
8a953620 MD	169	while (goflag == GOFLAG_INIT)
	170	poll(NULL, 0, 1);
	171	while (goflag == GOFLAG_RUN) {
	172	synchronize_rcu();
	173	n_updates_local++;
	174	}
	175	__get_thread_var(n_updates_pt) += n_updates_local;
b0b31506	176	return NULL;
8a953620 MD	177	}
	178
	179	void perftestinit(void)
	180	{
	181	init_per_thread(n_reads_pt, 0LL);
	182	init_per_thread(n_updates_pt, 0LL);
ec4e58a3	183	uatomic_set(&nthreadsrunning, 0);
8a953620 MD	184	}
	185
	186	void perftestrun(int nthreads, int nreaders, int nupdaters)
	187	{
	188	int t;
	189	int duration = 1;
	190
5481ddb3	191	cmm_smp_mb();
ec4e58a3	192	while (uatomic_read(&nthreadsrunning) < nthreads)
8a953620 MD	193	poll(NULL, 0, 1);
8a953620 MD	194	goflag = GOFLAG_RUN;
5481ddb3	195	cmm_smp_mb();
8a953620	196	sleep(duration);
5481ddb3	197	cmm_smp_mb();
8a953620	198	goflag = GOFLAG_STOP;
5481ddb3	199	cmm_smp_mb();
8a953620 MD	200	wait_all_threads();
	201	for_each_thread(t) {
	202	n_reads += per_thread(n_reads_pt, t);
	203	n_updates += per_thread(n_updates_pt, t);
	204	}
	205	printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n",
	206	n_reads, n_updates, nreaders, nupdaters, duration);
	207	printf("ns/read: %g ns/update: %g\n",
	208	((duration * 100010001000.*(double)nreaders) /
	209	(double)n_reads),
	210	((duration * 100010001000.*(double)nupdaters) /
	211	(double)n_updates));
7106ddf8 PM	212	if (get_cpu_call_rcu_data(0)) {
	213	fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n");
	214	free_all_cpu_call_rcu_data();
	215	}
8a953620 MD	216	exit(0);
	217	}
	218
	219	void perftest(int nreaders, int cpustride)
	220	{
	221	int i;
	222	long arg;
	223
	224	perftestinit();
	225	for (i = 0; i < nreaders; i++) {
	226	arg = (long)(i * cpustride);
	227	create_thread(rcu_read_perf_test, (void *)arg);
	228	}
	229	arg = (long)(i * cpustride);
	230	create_thread(rcu_update_perf_test, (void *)arg);
	231	perftestrun(i + 1, nreaders, 1);
	232	}
	233
	234	void rperftest(int nreaders, int cpustride)
	235	{
	236	int i;
	237	long arg;
	238
	239	perftestinit();
	240	init_per_thread(n_reads_pt, 0LL);
	241	for (i = 0; i < nreaders; i++) {
	242	arg = (long)(i * cpustride);
	243	create_thread(rcu_read_perf_test, (void *)arg);
	244	}
	245	perftestrun(i, nreaders, 0);
	246	}
	247
	248	void uperftest(int nupdaters, int cpustride)
	249	{
	250	int i;
	251	long arg;
	252
	253	perftestinit();
	254	init_per_thread(n_reads_pt, 0LL);
	255	for (i = 0; i < nupdaters; i++) {
	256	arg = (long)(i * cpustride);
	257	create_thread(rcu_update_perf_test, (void *)arg);
	258	}
	259	perftestrun(i, 0, nupdaters);
	260	}
	261
	262	/*
	263	* Stress test.
	264	*/
	265
	266	#define RCU_STRESS_PIPE_LEN 10
	267
	268	struct rcu_stress {
	269	int pipe_count;
	270	int mbtest;
	271	};
	272
b0b31506	273	struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
8a953620 MD	274	struct rcu_stress *rcu_stress_current;
	275	int rcu_stress_idx = 0;
	276
	277	int n_mberror = 0;
	278	DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
	279
	280	int garbage = 0;
	281
	282	void rcu_read_stress_test(void arg)
	283	{
	284	int i;
	285	int itercnt = 0;
	286	struct rcu_stress *p;
	287	int pc;
	288
121a5d44	289	rcu_register_thread();
3bf02b5b	290	put_thread_offline();
8a953620 MD	291	while (goflag == GOFLAG_INIT)
8a953620 MD	292	poll(NULL, 0, 1);
3bf02b5b	293	put_thread_online();
8a953620 MD	294	while (goflag == GOFLAG_RUN) {
	295	rcu_read_lock();
	296	p = rcu_dereference(rcu_stress_current);
	297	if (p->mbtest == 0)
	298	n_mberror++;
	299	rcu_read_lock_nest();
	300	for (i = 0; i < 100; i++)
	301	garbage++;
	302	rcu_read_unlock_nest();
	303	pc = p->pipe_count;
	304	rcu_read_unlock();
	305	if ((pc > RCU_STRESS_PIPE_LEN) \|\| (pc < 0))
	306	pc = RCU_STRESS_PIPE_LEN;
	307	__get_thread_var(rcu_stress_count)[pc]++;
	308	__get_thread_var(n_reads_pt)++;
	309	mark_rcu_quiescent_state();
	310	if ((++itercnt % 0x1000) == 0) {
	311	put_thread_offline();
	312	put_thread_online_delay();
	313	put_thread_online();
	314	}
	315	}
	316	put_thread_offline();
121a5d44	317	rcu_unregister_thread();
8a953620 MD	318
	319	return (NULL);
	320	}
	321
b57aee66 PM	322	static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
	323	static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
	324
	325	void rcu_update_stress_test_rcu(struct rcu_head *head)
	326	{
	327	if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) {
	328	perror("pthread_mutex_lock");
	329	exit(-1);
	330	}
	331	if (pthread_cond_signal(&call_rcu_test_cond) != 0) {
	332	perror("pthread_cond_signal");
	333	exit(-1);
	334	}
	335	if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) {
	336	perror("pthread_mutex_unlock");
	337	exit(-1);
	338	}
	339	}
	340
8a953620 MD	341	void rcu_update_stress_test(void arg)
	342	{
	343	int i;
	344	struct rcu_stress *p;
b57aee66	345	struct rcu_head rh;
8a953620 MD	346
	347	while (goflag == GOFLAG_INIT)
	348	poll(NULL, 0, 1);
	349	while (goflag == GOFLAG_RUN) {
	350	i = rcu_stress_idx + 1;
	351	if (i >= RCU_STRESS_PIPE_LEN)
	352	i = 0;
	353	p = &rcu_stress_array[i];
	354	p->mbtest = 0;
5481ddb3	355	cmm_smp_mb();
8a953620 MD	356	p->pipe_count = 0;
	357	p->mbtest = 1;
	358	rcu_assign_pointer(rcu_stress_current, p);
	359	rcu_stress_idx = i;
	360	for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
	361	if (i != rcu_stress_idx)
	362	rcu_stress_array[i].pipe_count++;
b57aee66 PM	363	if (n_updates & 0x1)
	364	synchronize_rcu();
	365	else {
	366	if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) {
	367	perror("pthread_mutex_lock");
	368	exit(-1);
	369	}
8bf9d59b	370	rcu_register_thread();
b57aee66	371	call_rcu(&rh, rcu_update_stress_test_rcu);
8bf9d59b MD	372	rcu_unregister_thread();
	373	/*
	374	* Our MacOS X test machine with the following
	375	* config:
	376	* 15.6.0 Darwin Kernel Version 15.6.0
	377	* root:xnu-3248.60.10~1/RELEASE_X86_64
	378	* appears to have issues with liburcu-signal
	379	* signal being delivered on top of
	380	* pthread_cond_wait. It seems to make the
	381	* thread continue, and therefore corrupt the
	382	* rcu_head. Work around this issue by
	383	* unregistering the RCU read-side thread
	384	* immediately after call_rcu (call_rcu needs
	385	* us to be registered RCU readers).
	386	*/
b57aee66 PM	387	if (pthread_cond_wait(&call_rcu_test_cond,
	388	&call_rcu_test_mutex) != 0) {
	389	perror("pthread_cond_wait");
	390	exit(-1);
	391	}
	392	if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) {
	393	perror("pthread_mutex_unlock");
	394	exit(-1);
	395	}
	396	}
8a953620 MD	397	n_updates++;
8a953620 MD	398	}
5e92ab62	399
b0b31506	400	return NULL;
8a953620 MD	401	}
	402
	403	void rcu_fake_update_stress_test(void arg)
	404	{
b57aee66 PM	405	if ((random() & 0xf00) == 0) {
	406	struct call_rcu_data *crdp;
	407
c1d2c60b	408	crdp = create_call_rcu_data(0, -1);
b57aee66 PM	409	if (crdp != NULL) {
	410	fprintf(stderr,
	411	"Using per-thread call_rcu() worker.\n");
	412	set_thread_call_rcu_data(crdp);
	413	}
	414	}
8a953620 MD	415	while (goflag == GOFLAG_INIT)
	416	poll(NULL, 0, 1);
	417	while (goflag == GOFLAG_RUN) {
	418	synchronize_rcu();
	419	poll(NULL, 0, 1);
	420	}
b0b31506	421	return NULL;
8a953620 MD	422	}
	423
	424	void stresstest(int nreaders)
	425	{
	426	int i;
	427	int t;
	428	long long *p;
	429	long long sum;
	430
	431	init_per_thread(n_reads_pt, 0LL);
	432	for_each_thread(t) {
	433	p = &per_thread(rcu_stress_count,t)[0];
	434	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
	435	p[i] = 0LL;
	436	}
	437	rcu_stress_current = &rcu_stress_array[0];
	438	rcu_stress_current->pipe_count = 0;
	439	rcu_stress_current->mbtest = 1;
	440	for (i = 0; i < nreaders; i++)
	441	create_thread(rcu_read_stress_test, NULL);
	442	create_thread(rcu_update_stress_test, NULL);
	443	for (i = 0; i < 5; i++)
	444	create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3	445	cmm_smp_mb();
8a953620	446	goflag = GOFLAG_RUN;
5481ddb3	447	cmm_smp_mb();
8a953620	448	sleep(10);
5481ddb3	449	cmm_smp_mb();
8a953620	450	goflag = GOFLAG_STOP;
5481ddb3	451	cmm_smp_mb();
8a953620 MD	452	wait_all_threads();
	453	for_each_thread(t)
	454	n_reads += per_thread(n_reads_pt, t);
b0b31506	455	printf("n_reads: %lld n_updates: %ld n_mberror: %d\n",
8a953620 MD	456	n_reads, n_updates, n_mberror);
	457	printf("rcu_stress_count:");
	458	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
	459	sum = 0LL;
	460	for_each_thread(t) {
	461	sum += per_thread(rcu_stress_count, t)[i];
	462	}
	463	printf(" %lld", sum);
	464	}
	465	printf("\n");
7106ddf8 PM	466	if (get_cpu_call_rcu_data(0)) {
	467	fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n");
	468	free_all_cpu_call_rcu_data();
	469	}
8a953620 MD	470	exit(0);
	471	}
	472
	473	/*
	474	* Mainprogram.
	475	*/
	476
	477	void usage(int argc, char *argv[])
	478	{
	479	fprintf(stderr, "Usage: %s [nreaders [ perf \| stress ] ]\n", argv[0]);
	480	exit(-1);
	481	}
	482
	483	int main(int argc, char *argv[])
	484	{
	485	int nreaders = 1;
	486	int cpustride = 1;
	487
	488	smp_init();
	489	//rcu_init();
b57aee66 PM	490	srandom(time(NULL));
	491	if (random() & 0x100) {
	492	fprintf(stderr, "Allocating per-CPU call_rcu threads.\n");
	493	if (create_all_cpu_call_rcu_data(0))
	494	perror("create_all_cpu_call_rcu_data");
	495	}
8a953620	496
9b171f46 MD	497	#ifdef DEBUG_YIELD
	498	yield_active \|= YIELD_READ;
	499	yield_active \|= YIELD_WRITE;
	500	#endif
	501
8a953620 MD	502	if (argc > 1) {
	503	nreaders = strtoul(argv[1], NULL, 0);
	504	if (argc == 2)
	505	perftest(nreaders, cpustride);
	506	if (argc > 3)
	507	cpustride = strtoul(argv[3], NULL, 0);
	508	if (strcmp(argv[2], "perf") == 0)
	509	perftest(nreaders, cpustride);
	510	else if (strcmp(argv[2], "rperf") == 0)
	511	rperftest(nreaders, cpustride);
	512	else if (strcmp(argv[2], "uperf") == 0)
	513	uperftest(nreaders, cpustride);
	514	else if (strcmp(argv[2], "stress") == 0)
	515	stresstest(nreaders);
	516	usage(argc, argv);
	517	}
	518	perftest(nreaders, cpustride);
0578089f	519	return 0;
8a953620	520	}