[urcu.git] / tests / regression / rcutorture.h

/*
 * rcutorture.h: simple user-level performance/stress test of RCU.
 *
 * Usage:
 * 	./rcu <nreaders> rperf [ <cpustride> ]
 * 		Run a read-side performance test with the specified
 * 		number of readers spaced by <cpustride>.
 * 		Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
 * 		CPUs from 0 to 30.
 * 	./rcu <nupdaters> uperf [ <cpustride> ]
 * 		Run an update-side performance test with the specified
 * 		number of updaters and specified CPU spacing.
 * 	./rcu <nreaders> perf [ <cpustride> ]
 * 		Run a combined read/update performance test with the specified
 * 		number of readers and one updater and specified CPU spacing.
 * 		The readers run on the low-numbered CPUs and the updater
 * 		of the highest-numbered CPU.
 *
 * The above tests produce output as follows:
 *
 * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
 * ns/read: 43.4707  ns/update: 6848.1
 *
 * The first line lists the total number of RCU reads and updates executed
 * during the test, the number of reader threads, the number of updater
 * threads, and the duration of the test in seconds.  The second line
 * lists the average duration of each type of operation in nanoseconds,
 * or "nan" if the corresponding type of operation was not performed.
 *
 * 	./rcu <nreaders> stress
 * 		Run a stress test with the specified number of readers and
 * 		one updater.  None of the threads are affinitied to any
 * 		particular CPU.
 *
 * This test produces output as follows:
 *
 * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
 *
 * The first line lists the number of RCU read and update operations
 * executed, followed by the number of memory-ordering violations
 * (which will be zero in a correct RCU implementation).  The second
 * line lists the number of readers observing progressively more stale
 * data.  A correct RCU implementation will have all but the first two
 * numbers non-zero.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
 */

/*
 * Test variables.
 */

#include <stdlib.h>
#include "tap.h"

#define NR_TESTS	1

DEFINE_PER_THREAD(long long, n_reads_pt);
DEFINE_PER_THREAD(long long, n_updates_pt);

enum callrcu_type {
	CALLRCU_GLOBAL,
	CALLRCU_PERCPU,
	CALLRCU_PERTHREAD,
};

static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;

long long n_reads = 0LL;
long n_updates = 0L;
int nthreadsrunning;
char argsbuf[64];

#define GOFLAG_INIT 0
#define GOFLAG_RUN  1
#define GOFLAG_STOP 2

volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
        = GOFLAG_INIT;

#define RCU_READ_RUN 1000

//MD
#define RCU_READ_NESTABLE

#ifdef RCU_READ_NESTABLE
#define rcu_read_lock_nest() rcu_read_lock()
#define rcu_read_unlock_nest() rcu_read_unlock()
#else /* #ifdef RCU_READ_NESTABLE */
#define rcu_read_lock_nest()
#define rcu_read_unlock_nest()
#endif /* #else #ifdef RCU_READ_NESTABLE */

#ifdef TORTURE_QSBR
#define mark_rcu_quiescent_state	rcu_quiescent_state
#define put_thread_offline		rcu_thread_offline
#define put_thread_online		rcu_thread_online
#endif

#ifndef mark_rcu_quiescent_state
#define mark_rcu_quiescent_state() do {} while (0)
#endif /* #ifdef mark_rcu_quiescent_state */

#ifndef put_thread_offline
#define put_thread_offline()		do {} while (0)
#define put_thread_online()		do {} while (0)
#define put_thread_online_delay()	do {} while (0)
#else /* #ifndef put_thread_offline */
#define put_thread_online_delay()	synchronize_rcu()
#endif /* #else #ifndef put_thread_offline */

/*
 * Performance test.
 */

static
void *rcu_read_perf_test(void *arg)
{
	int i;
	int me = (long)arg;
	long long n_reads_local = 0;

	rcu_register_thread();
	run_on(me);
	uatomic_inc(&nthreadsrunning);
	put_thread_offline();
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	put_thread_online();
	while (goflag == GOFLAG_RUN) {
		for (i = 0; i < RCU_READ_RUN; i++) {
			rcu_read_lock();
			/* rcu_read_lock_nest(); */
			/* rcu_read_unlock_nest(); */
			rcu_read_unlock();
		}
		n_reads_local += RCU_READ_RUN;
		mark_rcu_quiescent_state();
	}
	__get_thread_var(n_reads_pt) += n_reads_local;
	put_thread_offline();
	rcu_unregister_thread();

	return (NULL);
}

static
void *rcu_update_perf_test(void *arg)
{
	long long n_updates_local = 0;

	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = create_call_rcu_data(0, -1);
		if (crdp != NULL) {
			diag("Successfully using per-thread call_rcu() worker.");
			set_thread_call_rcu_data(crdp);
		}
	}
	uatomic_inc(&nthreadsrunning);
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		synchronize_rcu();
		n_updates_local++;
	}
	__get_thread_var(n_updates_pt) += n_updates_local;
	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = get_thread_call_rcu_data();
		set_thread_call_rcu_data(NULL);
		call_rcu_data_free(crdp);
	}
	return NULL;
}

static
void perftestinit(void)
{
	init_per_thread(n_reads_pt, 0LL);
	init_per_thread(n_updates_pt, 0LL);
	uatomic_set(&nthreadsrunning, 0);
}

static
int perftestrun(int nthreads, int nreaders, int nupdaters)
{
	int t;
	int duration = 1;

	cmm_smp_mb();
	while (uatomic_read(&nthreadsrunning) < nthreads)
		(void) poll(NULL, 0, 1);
	goflag = GOFLAG_RUN;
	cmm_smp_mb();
	sleep(duration);
	cmm_smp_mb();
	goflag = GOFLAG_STOP;
	cmm_smp_mb();
	wait_all_threads();
	for_each_thread(t) {
		n_reads += per_thread(n_reads_pt, t);
		n_updates += per_thread(n_updates_pt, t);
	}
	diag("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d",
	       n_reads, n_updates, nreaders, nupdaters, duration);
	diag("ns/read: %g  ns/update: %g",
	       ((duration * 1000*1000*1000.*(double)nreaders) /
	        (double)n_reads),
	       ((duration * 1000*1000*1000.*(double)nupdaters) /
	        (double)n_updates));
	if (get_cpu_call_rcu_data(0)) {
		diag("Deallocating per-CPU call_rcu threads.\n");
		free_all_cpu_call_rcu_data();
	}
	return 0;
}

static
int perftest(int nreaders, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	for (i = 0; i < nreaders; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_read_perf_test, (void *)arg);
	}
	arg = (long)(i * cpustride);
	create_thread(rcu_update_perf_test, (void *)arg);
	return perftestrun(i + 1, nreaders, 1);
}

static
int rperftest(int nreaders, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	init_per_thread(n_reads_pt, 0LL);
	for (i = 0; i < nreaders; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_read_perf_test, (void *)arg);
	}
	return perftestrun(i, nreaders, 0);
}

static
int uperftest(int nupdaters, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	init_per_thread(n_reads_pt, 0LL);
	for (i = 0; i < nupdaters; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_update_perf_test, (void *)arg);
	}
	return perftestrun(i, 0, nupdaters);
}

/*
 * Stress test.
 */

#define RCU_STRESS_PIPE_LEN 10

struct rcu_stress {
	int pipe_count;
	int mbtest;
};

struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
struct rcu_stress *rcu_stress_current;
int rcu_stress_idx = 0;

int n_mberror = 0;
DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);

int garbage = 0;

static
void *rcu_read_stress_test(void *arg)
{
	int i;
	int itercnt = 0;
	struct rcu_stress *p;
	int pc;

	rcu_register_thread();
	put_thread_offline();
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	put_thread_online();
	while (goflag == GOFLAG_RUN) {
		rcu_read_lock();
		p = rcu_dereference(rcu_stress_current);
		if (p->mbtest == 0)
			n_mberror++;
		rcu_read_lock_nest();
		for (i = 0; i < 100; i++)
			garbage++;
		rcu_read_unlock_nest();
		pc = p->pipe_count;
		rcu_read_unlock();
		if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
			pc = RCU_STRESS_PIPE_LEN;
		__get_thread_var(rcu_stress_count)[pc]++;
		__get_thread_var(n_reads_pt)++;
		mark_rcu_quiescent_state();
		if ((++itercnt % 0x1000) == 0) {
			put_thread_offline();
			put_thread_online_delay();
			put_thread_online();
		}
	}
	put_thread_offline();
	rcu_unregister_thread();

	return (NULL);
}

static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;

static
void rcu_update_stress_test_rcu(struct rcu_head *head)
{
	int ret;

	ret = pthread_mutex_lock(&call_rcu_test_mutex);
	if (ret) {
		errno = ret;
		diag("pthread_mutex_lock: %s",
			strerror(errno));
		abort();
	}
	ret = pthread_cond_signal(&call_rcu_test_cond);
	if (ret) {
		errno = ret;
		diag("pthread_cond_signal: %s",
			strerror(errno));
		abort();
	}
	ret = pthread_mutex_unlock(&call_rcu_test_mutex);
	if (ret) {
		errno = ret;
		diag("pthread_mutex_unlock: %s",
			strerror(errno));
		abort();
	}
}

static
void *rcu_update_stress_test(void *arg)
{
	int i;
	struct rcu_stress *p;
	struct rcu_head rh;

	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		i = rcu_stress_idx + 1;
		if (i >= RCU_STRESS_PIPE_LEN)
			i = 0;
		p = &rcu_stress_array[i];
		p->mbtest = 0;
		cmm_smp_mb();
		p->pipe_count = 0;
		p->mbtest = 1;
		rcu_assign_pointer(rcu_stress_current, p);
		rcu_stress_idx = i;
		for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
			if (i != rcu_stress_idx)
				rcu_stress_array[i].pipe_count++;
		if (n_updates & 0x1)
			synchronize_rcu();
		else {
			int ret;

			ret = pthread_mutex_lock(&call_rcu_test_mutex);
			if (ret) {
				errno = ret;
				diag("pthread_mutex_lock: %s",
					strerror(errno));
				abort();
			}
			rcu_register_thread();
			call_rcu(&rh, rcu_update_stress_test_rcu);
			rcu_unregister_thread();
			/*
			 * Our MacOS X test machine with the following
			 * config:
			 * 15.6.0 Darwin Kernel Version 15.6.0
			 * root:xnu-3248.60.10~1/RELEASE_X86_64
			 * appears to have issues with liburcu-signal
			 * signal being delivered on top of
			 * pthread_cond_wait. It seems to make the
			 * thread continue, and therefore corrupt the
			 * rcu_head. Work around this issue by
			 * unregistering the RCU read-side thread
			 * immediately after call_rcu (call_rcu needs
			 * us to be registered RCU readers).
			 */
			ret = pthread_cond_wait(&call_rcu_test_cond,
					&call_rcu_test_mutex);
			if (ret) {
				errno = ret;
				diag("pthread_cond_signal: %s",
					strerror(errno));
				abort();
			}
			ret = pthread_mutex_unlock(&call_rcu_test_mutex);
			if (ret) {
				errno = ret;
				diag("pthread_mutex_unlock: %s",
					strerror(errno));
				abort();
			}
		}
		n_updates++;
	}

	return NULL;
}

static
void *rcu_fake_update_stress_test(void *arg)
{
	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = create_call_rcu_data(0, -1);
		if (crdp != NULL) {
			diag("Successfully using per-thread call_rcu() worker.");
			set_thread_call_rcu_data(crdp);
		}
	}
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		synchronize_rcu();
		(void) poll(NULL, 0, 1);
	}
	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = get_thread_call_rcu_data();
		set_thread_call_rcu_data(NULL);
		call_rcu_data_free(crdp);
	}
	return NULL;
}

static
int stresstest(int nreaders)
{
	int i;
	int t;
	long long *p;
	long long sum;

	init_per_thread(n_reads_pt, 0LL);
	for_each_thread(t) {
		p = &per_thread(rcu_stress_count,t)[0];
		for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
			p[i] = 0LL;
	}
	rcu_stress_current = &rcu_stress_array[0];
	rcu_stress_current->pipe_count = 0;
	rcu_stress_current->mbtest = 1;
	for (i = 0; i < nreaders; i++)
		create_thread(rcu_read_stress_test, NULL);
	create_thread(rcu_update_stress_test, NULL);
	for (i = 0; i < 5; i++)
		create_thread(rcu_fake_update_stress_test, NULL);
	cmm_smp_mb();
	goflag = GOFLAG_RUN;
	cmm_smp_mb();
	sleep(10);
	cmm_smp_mb();
	goflag = GOFLAG_STOP;
	cmm_smp_mb();
	wait_all_threads();
	for_each_thread(t)
		n_reads += per_thread(n_reads_pt, t);
	diag("n_reads: %lld  n_updates: %ld  n_mberror: %d",
	       n_reads, n_updates, n_mberror);
	rdiag_start();
	rdiag("rcu_stress_count:");
	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
		sum = 0LL;
		for_each_thread(t) {
			sum += per_thread(rcu_stress_count, t)[i];
		}
		rdiag(" %lld", sum);
	}
	rdiag_end();
	if (get_cpu_call_rcu_data(0)) {
		diag("Deallocating per-CPU call_rcu threads.");
		free_all_cpu_call_rcu_data();
	}
	if (!n_mberror)
		return 0;
	else
		return -1;
}

/*
 * Mainprogram.
 */

static
void usage(int argc, char *argv[])
{
	diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
	exit(-1);
}

int main(int argc, char *argv[])
{
	int nreaders = 1;
	int cpustride = 1;

	plan_tests(NR_TESTS);

	smp_init();
	//rcu_init();
	if (argc > 4) {
		const char *callrcu_str = argv[4];;

		if (strcmp(callrcu_str, "callrcu_global") == 0) {
			callrcu_type = CALLRCU_GLOBAL;
		} else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
			callrcu_type = CALLRCU_PERCPU;
		} else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
			callrcu_type = CALLRCU_PERTHREAD;
		} else {
			usage(argc, argv);
			goto end;
		}
	}

	switch (callrcu_type) {
	case CALLRCU_GLOBAL:
		diag("Using global per-process call_rcu thread.");
		break;
	case CALLRCU_PERCPU:
		diag("Using per-CPU call_rcu threads.");
		if (create_all_cpu_call_rcu_data(0))
			diag("create_all_cpu_call_rcu_data: %s",
				strerror(errno));
		break;
	case CALLRCU_PERTHREAD:
		diag("Using per-thread call_rcu() worker.");
		break;
	default:
		abort();
	}

#ifdef DEBUG_YIELD
	yield_active |= YIELD_READ;
	yield_active |= YIELD_WRITE;
#endif

	if (argc > 1) {
		if (strcmp(argv[1], "-h") == 0
				|| strcmp(argv[1], "--help") == 0) {
			usage(argc, argv);
			goto end;
		}
		nreaders = strtoul(argv[1], NULL, 0);
		if (argc == 2) {
			ok(!perftest(nreaders, cpustride),
				"perftest readers: %d, stride: %d",
				nreaders, cpustride);
			goto end;
		}
		if (argc > 3)
			cpustride = strtoul(argv[3], NULL, 0);
		if (strcmp(argv[2], "perf") == 0)
			ok(!perftest(nreaders, cpustride),
				"perftest readers: %d, stride: %d",
				nreaders, cpustride);
		else if (strcmp(argv[2], "rperf") == 0)
			ok(!rperftest(nreaders, cpustride),
				"rperftest readers: %d, stride: %d",
				nreaders, cpustride);
		else if (strcmp(argv[2], "uperf") == 0)
			ok(!uperftest(nreaders, cpustride),
				"uperftest readers: %d, stride: %d",
				nreaders, cpustride);
		else if (strcmp(argv[2], "stress") == 0)
			ok(!stresstest(nreaders),
				"stresstest readers: %d, stride: %d",
				nreaders, cpustride);
		else
			usage(argc, argv);
	} else {
		usage(argc, argv);
	}
end:
	return exit_status();
}
Commit	Line	Data
8a953620 MD	1	/*
	2	* rcutorture.h: simple user-level performance/stress test of RCU.
	3	*
	4	* Usage:
	5	* ./rcu <nreaders> rperf [ <cpustride> ]
	6	* Run a read-side performance test with the specified
	7	* number of readers spaced by <cpustride>.
	8	* Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
	9	* CPUs from 0 to 30.
	10	* ./rcu <nupdaters> uperf [ <cpustride> ]
	11	* Run an update-side performance test with the specified
	12	* number of updaters and specified CPU spacing.
	13	* ./rcu <nreaders> perf [ <cpustride> ]
	14	* Run a combined read/update performance test with the specified
	15	* number of readers and one updater and specified CPU spacing.
	16	* The readers run on the low-numbered CPUs and the updater
	17	* of the highest-numbered CPU.
	18	*
	19	* The above tests produce output as follows:
	20	*
	21	* n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
	22	* ns/read: 43.4707 ns/update: 6848.1
	23	*
	24	* The first line lists the total number of RCU reads and updates executed
	25	* during the test, the number of reader threads, the number of updater
	26	* threads, and the duration of the test in seconds. The second line
	27	* lists the average duration of each type of operation in nanoseconds,
	28	* or "nan" if the corresponding type of operation was not performed.
	29	*
	30	* ./rcu <nreaders> stress
	31	* Run a stress test with the specified number of readers and
	32	* one updater. None of the threads are affinitied to any
	33	* particular CPU.
	34	*
	35	* This test produces output as follows:
	36	*
	37	* n_reads: 114633217 n_updates: 3903415 n_mberror: 0
	38	* rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
	39	*
	40	* The first line lists the number of RCU read and update operations
	41	* executed, followed by the number of memory-ordering violations
	42	* (which will be zero in a correct RCU implementation). The second
	43	* line lists the number of readers observing progressively more stale
	44	* data. A correct RCU implementation will have all but the first two
	45	* numbers non-zero.
	46	*
	47	* This program is free software; you can redistribute it and/or modify
	48	* it under the terms of the GNU General Public License as published by
	49	* the Free Software Foundation; either version 2 of the License, or
	50	* (at your option) any later version.
	51	*
	52	* This program is distributed in the hope that it will be useful,
	53	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	54	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	55	* GNU General Public License for more details.
	56	*
	57	* You should have received a copy of the GNU General Public License
	58	* along with this program; if not, write to the Free Software
3282a76b	59	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
8a953620 MD	60	*
	61	* Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
	62	*/
	63
	64	/*
	65	* Test variables.
	66	*/
	67
b57aee66	68	#include <stdlib.h>
ad460058 MD	69	#include "tap.h"
	70
	71	#define NR_TESTS 1
b57aee66	72
8a953620 MD	73	DEFINE_PER_THREAD(long long, n_reads_pt);
	74	DEFINE_PER_THREAD(long long, n_updates_pt);
	75
26b5a74b MD	76	enum callrcu_type {
	77	CALLRCU_GLOBAL,
	78	CALLRCU_PERCPU,
	79	CALLRCU_PERTHREAD,
	80	};
	81
	82	static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
	83
8a953620 MD	84	long long n_reads = 0LL;
8a953620 MD	85	long n_updates = 0L;
6ee91d83	86	int nthreadsrunning;
8a953620 MD	87	char argsbuf[64];
	88
	89	#define GOFLAG_INIT 0
	90	#define GOFLAG_RUN 1
	91	#define GOFLAG_STOP 2
	92
4967f005 PB	93	volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
4967f005 PB	94	= GOFLAG_INIT;
8a953620 MD	95
	96	#define RCU_READ_RUN 1000
	97
	98	//MD
	99	#define RCU_READ_NESTABLE
	100
	101	#ifdef RCU_READ_NESTABLE
	102	#define rcu_read_lock_nest() rcu_read_lock()
	103	#define rcu_read_unlock_nest() rcu_read_unlock()
	104	#else /* #ifdef RCU_READ_NESTABLE */
	105	#define rcu_read_lock_nest()
	106	#define rcu_read_unlock_nest()
	107	#endif /* #else #ifdef RCU_READ_NESTABLE */
	108
1a43bbd8 MD	109	#ifdef TORTURE_QSBR
	110	#define mark_rcu_quiescent_state rcu_quiescent_state
	111	#define put_thread_offline rcu_thread_offline
	112	#define put_thread_online rcu_thread_online
	113	#endif
	114
8a953620	115	#ifndef mark_rcu_quiescent_state
447c9339	116	#define mark_rcu_quiescent_state() do {} while (0)
8a953620 MD	117	#endif /* #ifdef mark_rcu_quiescent_state */
	118
	119	#ifndef put_thread_offline
447c9339 MJ	120	#define put_thread_offline() do {} while (0)
	121	#define put_thread_online() do {} while (0)
	122	#define put_thread_online_delay() do {} while (0)
8a953620 MD	123	#else /* #ifndef put_thread_offline */
	124	#define put_thread_online_delay() synchronize_rcu()
	125	#endif /* #else #ifndef put_thread_offline */
	126
	127	/*
	128	* Performance test.
	129	*/
	130
61c3fb60	131	static
8a953620 MD	132	void rcu_read_perf_test(void arg)
	133	{
	134	int i;
	135	int me = (long)arg;
8a953620 MD	136	long long n_reads_local = 0;
8a953620 MD	137
121a5d44	138	rcu_register_thread();
8a953620	139	run_on(me);
ec4e58a3	140	uatomic_inc(&nthreadsrunning);
3bf02b5b	141	put_thread_offline();
8a953620	142	while (goflag == GOFLAG_INIT)
775aff2e	143	(void) poll(NULL, 0, 1);
3bf02b5b	144	put_thread_online();
8a953620 MD	145	while (goflag == GOFLAG_RUN) {
	146	for (i = 0; i < RCU_READ_RUN; i++) {
	147	rcu_read_lock();
	148	/* rcu_read_lock_nest(); */
	149	/* rcu_read_unlock_nest(); */
	150	rcu_read_unlock();
	151	}
	152	n_reads_local += RCU_READ_RUN;
	153	mark_rcu_quiescent_state();
	154	}
	155	__get_thread_var(n_reads_pt) += n_reads_local;
	156	put_thread_offline();
121a5d44	157	rcu_unregister_thread();
8a953620 MD	158
	159	return (NULL);
	160	}
	161
61c3fb60	162	static
8a953620 MD	163	void rcu_update_perf_test(void arg)
	164	{
	165	long long n_updates_local = 0;
	166
26b5a74b	167	if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66 PM	168	struct call_rcu_data *crdp;
b57aee66 PM	169
c1d2c60b	170	crdp = create_call_rcu_data(0, -1);
b57aee66	171	if (crdp != NULL) {
26b5a74b	172	diag("Successfully using per-thread call_rcu() worker.");
b57aee66 PM	173	set_thread_call_rcu_data(crdp);
	174	}
	175	}
ec4e58a3	176	uatomic_inc(&nthreadsrunning);
8a953620	177	while (goflag == GOFLAG_INIT)
775aff2e	178	(void) poll(NULL, 0, 1);
8a953620 MD	179	while (goflag == GOFLAG_RUN) {
	180	synchronize_rcu();
	181	n_updates_local++;
	182	}
	183	__get_thread_var(n_updates_pt) += n_updates_local;
26b5a74b MD	184	if (callrcu_type == CALLRCU_PERTHREAD) {
	185	struct call_rcu_data *crdp;
	186
	187	crdp = get_thread_call_rcu_data();
	188	set_thread_call_rcu_data(NULL);
	189	call_rcu_data_free(crdp);
	190	}
b0b31506	191	return NULL;
8a953620 MD	192	}
8a953620 MD	193
61c3fb60	194	static
8a953620 MD	195	void perftestinit(void)
	196	{
	197	init_per_thread(n_reads_pt, 0LL);
	198	init_per_thread(n_updates_pt, 0LL);
ec4e58a3	199	uatomic_set(&nthreadsrunning, 0);
8a953620 MD	200	}
8a953620 MD	201
61c3fb60	202	static
ad460058	203	int perftestrun(int nthreads, int nreaders, int nupdaters)
8a953620 MD	204	{
	205	int t;
	206	int duration = 1;
	207
5481ddb3	208	cmm_smp_mb();
ec4e58a3	209	while (uatomic_read(&nthreadsrunning) < nthreads)
775aff2e	210	(void) poll(NULL, 0, 1);
8a953620	211	goflag = GOFLAG_RUN;
5481ddb3	212	cmm_smp_mb();
8a953620	213	sleep(duration);
5481ddb3	214	cmm_smp_mb();
8a953620	215	goflag = GOFLAG_STOP;
5481ddb3	216	cmm_smp_mb();
8a953620 MD	217	wait_all_threads();
	218	for_each_thread(t) {
	219	n_reads += per_thread(n_reads_pt, t);
	220	n_updates += per_thread(n_updates_pt, t);
	221	}
ad460058	222	diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
8a953620	223	n_reads, n_updates, nreaders, nupdaters, duration);
ad460058	224	diag("ns/read: %g ns/update: %g",
8a953620 MD	225	((duration * 100010001000.*(double)nreaders) /
	226	(double)n_reads),
	227	((duration * 100010001000.*(double)nupdaters) /
	228	(double)n_updates));
7106ddf8	229	if (get_cpu_call_rcu_data(0)) {
ad460058	230	diag("Deallocating per-CPU call_rcu threads.\n");
7106ddf8 PM	231	free_all_cpu_call_rcu_data();
7106ddf8 PM	232	}
ad460058	233	return 0;
8a953620 MD	234	}
8a953620 MD	235
61c3fb60	236	static
ad460058	237	int perftest(int nreaders, int cpustride)
8a953620 MD	238	{
	239	int i;
	240	long arg;
	241
	242	perftestinit();
	243	for (i = 0; i < nreaders; i++) {
	244	arg = (long)(i * cpustride);
	245	create_thread(rcu_read_perf_test, (void *)arg);
	246	}
	247	arg = (long)(i * cpustride);
	248	create_thread(rcu_update_perf_test, (void *)arg);
ad460058	249	return perftestrun(i + 1, nreaders, 1);
8a953620 MD	250	}
8a953620 MD	251
61c3fb60	252	static
ad460058	253	int rperftest(int nreaders, int cpustride)
8a953620 MD	254	{
	255	int i;
	256	long arg;
	257
	258	perftestinit();
	259	init_per_thread(n_reads_pt, 0LL);
	260	for (i = 0; i < nreaders; i++) {
	261	arg = (long)(i * cpustride);
	262	create_thread(rcu_read_perf_test, (void *)arg);
	263	}
ad460058	264	return perftestrun(i, nreaders, 0);
8a953620 MD	265	}
8a953620 MD	266
61c3fb60	267	static
ad460058	268	int uperftest(int nupdaters, int cpustride)
8a953620 MD	269	{
	270	int i;
	271	long arg;
	272
	273	perftestinit();
	274	init_per_thread(n_reads_pt, 0LL);
	275	for (i = 0; i < nupdaters; i++) {
	276	arg = (long)(i * cpustride);
	277	create_thread(rcu_update_perf_test, (void *)arg);
	278	}
ad460058	279	return perftestrun(i, 0, nupdaters);
8a953620 MD	280	}
	281
	282	/*
	283	* Stress test.
	284	*/
	285
	286	#define RCU_STRESS_PIPE_LEN 10
	287
	288	struct rcu_stress {
	289	int pipe_count;
	290	int mbtest;
	291	};
	292
b0b31506	293	struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
8a953620 MD	294	struct rcu_stress *rcu_stress_current;
	295	int rcu_stress_idx = 0;
	296
	297	int n_mberror = 0;
	298	DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
	299
	300	int garbage = 0;
	301
61c3fb60	302	static
8a953620 MD	303	void rcu_read_stress_test(void arg)
	304	{
	305	int i;
	306	int itercnt = 0;
	307	struct rcu_stress *p;
	308	int pc;
	309
121a5d44	310	rcu_register_thread();
3bf02b5b	311	put_thread_offline();
8a953620	312	while (goflag == GOFLAG_INIT)
775aff2e	313	(void) poll(NULL, 0, 1);
3bf02b5b	314	put_thread_online();
8a953620 MD	315	while (goflag == GOFLAG_RUN) {
	316	rcu_read_lock();
	317	p = rcu_dereference(rcu_stress_current);
	318	if (p->mbtest == 0)
	319	n_mberror++;
	320	rcu_read_lock_nest();
	321	for (i = 0; i < 100; i++)
	322	garbage++;
	323	rcu_read_unlock_nest();
	324	pc = p->pipe_count;
	325	rcu_read_unlock();
	326	if ((pc > RCU_STRESS_PIPE_LEN) \|\| (pc < 0))
	327	pc = RCU_STRESS_PIPE_LEN;
	328	__get_thread_var(rcu_stress_count)[pc]++;
	329	__get_thread_var(n_reads_pt)++;
	330	mark_rcu_quiescent_state();
	331	if ((++itercnt % 0x1000) == 0) {
	332	put_thread_offline();
	333	put_thread_online_delay();
	334	put_thread_online();
	335	}
	336	}
	337	put_thread_offline();
121a5d44	338	rcu_unregister_thread();
8a953620 MD	339
	340	return (NULL);
	341	}
	342
b57aee66 PM	343	static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
	344	static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
	345
61c3fb60	346	static
b57aee66 PM	347	void rcu_update_stress_test_rcu(struct rcu_head *head)
b57aee66 PM	348	{
ad460058 MD	349	int ret;
	350
	351	ret = pthread_mutex_lock(&call_rcu_test_mutex);
	352	if (ret) {
	353	errno = ret;
	354	diag("pthread_mutex_lock: %s",
	355	strerror(errno));
	356	abort();
b57aee66	357	}
ad460058 MD	358	ret = pthread_cond_signal(&call_rcu_test_cond);
	359	if (ret) {
	360	errno = ret;
	361	diag("pthread_cond_signal: %s",
	362	strerror(errno));
	363	abort();
b57aee66	364	}
ad460058 MD	365	ret = pthread_mutex_unlock(&call_rcu_test_mutex);
	366	if (ret) {
	367	errno = ret;
	368	diag("pthread_mutex_unlock: %s",
	369	strerror(errno));
	370	abort();
b57aee66 PM	371	}
	372	}
	373
61c3fb60	374	static
8a953620 MD	375	void rcu_update_stress_test(void arg)
	376	{
	377	int i;
	378	struct rcu_stress *p;
b57aee66	379	struct rcu_head rh;
8a953620 MD	380
8a953620 MD	381	while (goflag == GOFLAG_INIT)
775aff2e	382	(void) poll(NULL, 0, 1);
8a953620 MD	383	while (goflag == GOFLAG_RUN) {
	384	i = rcu_stress_idx + 1;
	385	if (i >= RCU_STRESS_PIPE_LEN)
	386	i = 0;
	387	p = &rcu_stress_array[i];
	388	p->mbtest = 0;
5481ddb3	389	cmm_smp_mb();
8a953620 MD	390	p->pipe_count = 0;
	391	p->mbtest = 1;
	392	rcu_assign_pointer(rcu_stress_current, p);
	393	rcu_stress_idx = i;
	394	for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
	395	if (i != rcu_stress_idx)
	396	rcu_stress_array[i].pipe_count++;
b57aee66 PM	397	if (n_updates & 0x1)
	398	synchronize_rcu();
	399	else {
ad460058 MD	400	int ret;
	401
	402	ret = pthread_mutex_lock(&call_rcu_test_mutex);
	403	if (ret) {
	404	errno = ret;
	405	diag("pthread_mutex_lock: %s",
	406	strerror(errno));
	407	abort();
b57aee66	408	}
0b9c513b	409	rcu_register_thread();
b57aee66	410	call_rcu(&rh, rcu_update_stress_test_rcu);
0b9c513b MD	411	rcu_unregister_thread();
	412	/*
	413	* Our MacOS X test machine with the following
	414	* config:
	415	* 15.6.0 Darwin Kernel Version 15.6.0
	416	* root:xnu-3248.60.10~1/RELEASE_X86_64
	417	* appears to have issues with liburcu-signal
	418	* signal being delivered on top of
	419	* pthread_cond_wait. It seems to make the
	420	* thread continue, and therefore corrupt the
	421	* rcu_head. Work around this issue by
	422	* unregistering the RCU read-side thread
	423	* immediately after call_rcu (call_rcu needs
	424	* us to be registered RCU readers).
	425	*/
ad460058 MD	426	ret = pthread_cond_wait(&call_rcu_test_cond,
	427	&call_rcu_test_mutex);
	428	if (ret) {
	429	errno = ret;
	430	diag("pthread_cond_signal: %s",
	431	strerror(errno));
	432	abort();
b57aee66	433	}
ad460058 MD	434	ret = pthread_mutex_unlock(&call_rcu_test_mutex);
	435	if (ret) {
	436	errno = ret;
	437	diag("pthread_mutex_unlock: %s",
	438	strerror(errno));
	439	abort();
b57aee66 PM	440	}
b57aee66 PM	441	}
8a953620 MD	442	n_updates++;
8a953620 MD	443	}
a13ef613	444
b0b31506	445	return NULL;
8a953620 MD	446	}
8a953620 MD	447
61c3fb60	448	static
8a953620 MD	449	void rcu_fake_update_stress_test(void arg)
8a953620 MD	450	{
26b5a74b	451	if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66 PM	452	struct call_rcu_data *crdp;
b57aee66 PM	453
c1d2c60b	454	crdp = create_call_rcu_data(0, -1);
b57aee66	455	if (crdp != NULL) {
26b5a74b	456	diag("Successfully using per-thread call_rcu() worker.");
b57aee66 PM	457	set_thread_call_rcu_data(crdp);
	458	}
	459	}
8a953620	460	while (goflag == GOFLAG_INIT)
775aff2e	461	(void) poll(NULL, 0, 1);
8a953620 MD	462	while (goflag == GOFLAG_RUN) {
8a953620 MD	463	synchronize_rcu();
775aff2e	464	(void) poll(NULL, 0, 1);
8a953620	465	}
26b5a74b MD	466	if (callrcu_type == CALLRCU_PERTHREAD) {
	467	struct call_rcu_data *crdp;
	468
	469	crdp = get_thread_call_rcu_data();
	470	set_thread_call_rcu_data(NULL);
	471	call_rcu_data_free(crdp);
	472	}
b0b31506	473	return NULL;
8a953620 MD	474	}
8a953620 MD	475
61c3fb60	476	static
ad460058	477	int stresstest(int nreaders)
8a953620 MD	478	{
	479	int i;
	480	int t;
	481	long long *p;
	482	long long sum;
	483
	484	init_per_thread(n_reads_pt, 0LL);
	485	for_each_thread(t) {
	486	p = &per_thread(rcu_stress_count,t)[0];
	487	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
	488	p[i] = 0LL;
	489	}
	490	rcu_stress_current = &rcu_stress_array[0];
	491	rcu_stress_current->pipe_count = 0;
	492	rcu_stress_current->mbtest = 1;
	493	for (i = 0; i < nreaders; i++)
	494	create_thread(rcu_read_stress_test, NULL);
	495	create_thread(rcu_update_stress_test, NULL);
	496	for (i = 0; i < 5; i++)
	497	create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3	498	cmm_smp_mb();
8a953620	499	goflag = GOFLAG_RUN;
5481ddb3	500	cmm_smp_mb();
8a953620	501	sleep(10);
5481ddb3	502	cmm_smp_mb();
8a953620	503	goflag = GOFLAG_STOP;
5481ddb3	504	cmm_smp_mb();
8a953620 MD	505	wait_all_threads();
	506	for_each_thread(t)
	507	n_reads += per_thread(n_reads_pt, t);
ad460058	508	diag("n_reads: %lld n_updates: %ld n_mberror: %d",
8a953620	509	n_reads, n_updates, n_mberror);
ad460058 MD	510	rdiag_start();
ad460058 MD	511	rdiag("rcu_stress_count:");
8a953620 MD	512	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
	513	sum = 0LL;
	514	for_each_thread(t) {
	515	sum += per_thread(rcu_stress_count, t)[i];
	516	}
ad460058	517	rdiag(" %lld", sum);
8a953620	518	}
ad460058	519	rdiag_end();
7106ddf8	520	if (get_cpu_call_rcu_data(0)) {
ad460058	521	diag("Deallocating per-CPU call_rcu threads.");
7106ddf8 PM	522	free_all_cpu_call_rcu_data();
7106ddf8 PM	523	}
ad460058 MD	524	if (!n_mberror)
	525	return 0;
	526	else
	527	return -1;
8a953620 MD	528	}
	529
	530	/*
	531	* Mainprogram.
	532	*/
	533
61c3fb60	534	static
8a953620 MD	535	void usage(int argc, char *argv[])
8a953620 MD	536	{
26b5a74b	537	diag("Usage: %s nreaders [ perf \| rperf \| uperf \| stress ] [ stride ] [ callrcu_global \| callrcu_percpu \| callrcu_perthread ]\n", argv[0]);
8a953620 MD	538	exit(-1);
	539	}
	540
	541	int main(int argc, char *argv[])
	542	{
	543	int nreaders = 1;
	544	int cpustride = 1;
	545
ad460058 MD	546	plan_tests(NR_TESTS);
ad460058 MD	547
8a953620 MD	548	smp_init();
8a953620 MD	549	//rcu_init();
26b5a74b MD	550	if (argc > 4) {
	551	const char *callrcu_str = argv[4];;
	552
	553	if (strcmp(callrcu_str, "callrcu_global") == 0) {
	554	callrcu_type = CALLRCU_GLOBAL;
	555	} else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
	556	callrcu_type = CALLRCU_PERCPU;
	557	} else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
	558	callrcu_type = CALLRCU_PERTHREAD;
	559	} else {
	560	usage(argc, argv);
	561	goto end;
	562	}
	563	}
929cfaff	564
26b5a74b MD	565	switch (callrcu_type) {
	566	case CALLRCU_GLOBAL:
	567	diag("Using global per-process call_rcu thread.");
	568	break;
	569	case CALLRCU_PERCPU:
	570	diag("Using per-CPU call_rcu threads.");
b57aee66	571	if (create_all_cpu_call_rcu_data(0))
ad460058 MD	572	diag("create_all_cpu_call_rcu_data: %s",
ad460058 MD	573	strerror(errno));
26b5a74b MD	574	break;
	575	case CALLRCU_PERTHREAD:
	576	diag("Using per-thread call_rcu() worker.");
	577	break;
	578	default:
	579	abort();
b57aee66	580	}
8a953620	581
9b171f46 MD	582	#ifdef DEBUG_YIELD
	583	yield_active \|= YIELD_READ;
	584	yield_active \|= YIELD_WRITE;
	585	#endif
	586
8a953620	587	if (argc > 1) {
26b5a74b MD	588	if (strcmp(argv[1], "-h") == 0
	589	\|\| strcmp(argv[1], "--help") == 0) {
	590	usage(argc, argv);
	591	goto end;
	592	}
8a953620	593	nreaders = strtoul(argv[1], NULL, 0);
ad460058 MD	594	if (argc == 2) {
	595	ok(!perftest(nreaders, cpustride),
	596	"perftest readers: %d, stride: %d",
	597	nreaders, cpustride);
	598	goto end;
	599	}
8a953620 MD	600	if (argc > 3)
	601	cpustride = strtoul(argv[3], NULL, 0);
	602	if (strcmp(argv[2], "perf") == 0)
ad460058 MD	603	ok(!perftest(nreaders, cpustride),
	604	"perftest readers: %d, stride: %d",
	605	nreaders, cpustride);
8a953620	606	else if (strcmp(argv[2], "rperf") == 0)
ad460058 MD	607	ok(!rperftest(nreaders, cpustride),
	608	"rperftest readers: %d, stride: %d",
	609	nreaders, cpustride);
8a953620	610	else if (strcmp(argv[2], "uperf") == 0)
ad460058 MD	611	ok(!uperftest(nreaders, cpustride),
	612	"uperftest readers: %d, stride: %d",
	613	nreaders, cpustride);
8a953620	614	else if (strcmp(argv[2], "stress") == 0)
ad460058 MD	615	ok(!stresstest(nreaders),
	616	"stresstest readers: %d, stride: %d",
	617	nreaders, cpustride);
	618	else
	619	usage(argc, argv);
	620	} else {
26b5a74b	621	usage(argc, argv);
8a953620	622	}
ad460058 MD	623	end:
ad460058 MD	624	return exit_status();
8a953620	625	}