[urcu.git] / urcu-bp.c

/*
 * urcu-bp.c
 *
 * Userspace RCU library, "bulletproof" version.
 *
 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
 */

#define _GNU_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <signal.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <poll.h>
#include <unistd.h>
#include <sys/mman.h>

#include "urcu/map/urcu-bp.h"

#include "urcu/static/urcu-bp.h"
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu-bp.h"

#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif

/* Sleep delay in us */
#define RCU_SLEEP_DELAY		1000
#define ARENA_INIT_ALLOC	16

void __attribute__((destructor)) rcu_bp_exit(void);

static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;

#ifdef DEBUG_YIELD
unsigned int yield_active;
unsigned int __thread rand_yield;
#endif

/*
 * Global grace period counter.
 * Contains the current RCU_GP_CTR_PHASE.
 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
 * Written to only by writer with mutex taken. Read by both writer and readers.
 */
long rcu_gp_ctr = RCU_GP_COUNT;

/*
 * Pointer to registry elements. Written to only by each individual reader. Read
 * by both the reader and the writers.
 */
struct rcu_reader __thread *rcu_reader;

static CDS_LIST_HEAD(registry);

struct registry_arena {
	void *p;
	size_t len;
	size_t used;
};

static struct registry_arena registry_arena;

/* Saved fork signal mask, protected by rcu_gp_lock */
static sigset_t saved_fork_signal_mask;

static void rcu_gc_registry(void);

static void mutex_lock(pthread_mutex_t *mutex)
{
	int ret;

#ifndef DISTRUST_SIGNALS_EXTREME
	ret = pthread_mutex_lock(mutex);
	if (ret) {
		perror("Error in pthread mutex lock");
		exit(-1);
	}
#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
		if (ret != EBUSY && ret != EINTR) {
			printf("ret = %d, errno = %d\n", ret, errno);
			perror("Error in pthread mutex lock");
			exit(-1);
		}
		poll(NULL,0,10);
	}
#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
}

static void mutex_unlock(pthread_mutex_t *mutex)
{
	int ret;

	ret = pthread_mutex_unlock(mutex);
	if (ret) {
		perror("Error in pthread mutex unlock");
		exit(-1);
	}
}

void update_counter_and_wait(void)
{
	CDS_LIST_HEAD(qsreaders);
	int wait_loops = 0;
	struct rcu_reader *index, *tmp;

	/* Switch parity: 0 -> 1, 1 -> 0 */
	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);

	/*
	 * Must commit qparity update to memory before waiting for other parity
	 * quiescent state. Failure to do so could result in the writer waiting
	 * forever while new readers are always accessing data (no progress).
	 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
	 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for each thread rcu_reader.ctr count to become 0.
	 */
	for (;;) {
		wait_loops++;
		cds_list_for_each_entry_safe(index, tmp, &registry, node) {
			if (!rcu_old_gp_ongoing(&index->ctr))
				cds_list_move(&index->node, &qsreaders);
		}

		if (cds_list_empty(&registry)) {
			break;
		} else {
			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
				usleep(RCU_SLEEP_DELAY);
			else
				caa_cpu_relax();
		}
	}
	/* put back the reader list in the registry */
	cds_list_splice(&qsreaders, &registry);
}

void synchronize_rcu(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);

	mutex_lock(&rcu_gp_lock);

	if (cds_list_empty(&registry))
		goto out;

	/* All threads should read qparity before accessing data structure
	 * where new ptr points to. */
	/* Write new ptr before changing the qparity */
	cmm_smp_mb();

	/* Remove old registry elements */
	rcu_gc_registry();

	/*
	 * Wait for previous parity to be empty of readers.
	 */
	update_counter_and_wait();	/* 0 -> 1, wait readers in parity 0 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for previous parity to be empty of readers.
	 */
	update_counter_and_wait();	/* 1 -> 0, wait readers in parity 1 */

	/*
	 * Finish waiting for reader threads before letting the old ptr being
	 * freed.
	 */
	cmm_smp_mb();
out:
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

/*
 * library wrappers to be used by non-LGPL compatible source code.
 */

void rcu_read_lock(void)
{
	_rcu_read_lock();
}

void rcu_read_unlock(void)
{
	_rcu_read_unlock();
}

/*
 * only grow for now.
 */
static void resize_arena(struct registry_arena *arena, size_t len)
{
	void *new_arena;

	if (!arena->p)
		new_arena = mmap(arena->p, len,
				 PROT_READ | PROT_WRITE,
				 MAP_ANONYMOUS | MAP_PRIVATE,
				 -1, 0);
	else
		new_arena = mremap(arena->p, arena->len,
				   len, MREMAP_MAYMOVE);
	assert(new_arena != MAP_FAILED);

	/*
	 * re-used the same region ?
	 */
	if (new_arena == arena->p)
		return;

	bzero(new_arena + arena->len, len - arena->len);
	arena->p = new_arena;
}

/* Called with signals off and mutex locked */
static void add_thread(void)
{
	struct rcu_reader *rcu_reader_reg;

	if (registry_arena.len
	    < registry_arena.used + sizeof(struct rcu_reader))
		resize_arena(&registry_arena,
		max(registry_arena.len << 1, ARENA_INIT_ALLOC));
	/*
	 * Find a free spot.
	 */
	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			break;
	}
	rcu_reader_reg->alloc = 1;
	registry_arena.used += sizeof(struct rcu_reader);

	/* Add to registry */
	rcu_reader_reg->tid = pthread_self();
	assert(rcu_reader_reg->ctr == 0);
	cds_list_add(&rcu_reader_reg->node, &registry);
	rcu_reader = rcu_reader_reg;
}

/* Called with signals off and mutex locked */
static void rcu_gc_registry(void)
{
	struct rcu_reader *rcu_reader_reg;
	pthread_t tid;
	int ret;

	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			continue;
		tid = rcu_reader_reg->tid;
		ret = pthread_kill(tid, 0);
		assert(ret != EINVAL);
		if (ret == ESRCH) {
			cds_list_del(&rcu_reader_reg->node);
			rcu_reader_reg->ctr = 0;
			rcu_reader_reg->alloc = 0;
			registry_arena.used -= sizeof(struct rcu_reader);
		}
	}
}

/* Disable signals, take mutex, add to registry */
void rcu_bp_register(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);

	/*
	 * Check if a signal concurrently registered our thread since
	 * the check in rcu_read_lock(). */
	if (rcu_reader)
		goto end;

	mutex_lock(&rcu_gp_lock);
	add_thread();
	mutex_unlock(&rcu_gp_lock);
end:
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_exit()
{
	munmap(registry_arena.p, registry_arena.len);
}

/*
 * Holding the rcu_gp_lock across fork will make sure we fork() don't race with
 * a concurrent thread executing with this same lock held. This ensures that the
 * registry is in a coherent state in the child.
 */
void rcu_bp_before_fork(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigemptyset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	assert(!ret);
	mutex_lock(&rcu_gp_lock);
	saved_fork_signal_mask = oldmask;
}

void rcu_bp_after_fork_parent(void)
{
	sigset_t oldmask;
	int ret;

	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_after_fork_child(void)
{
	sigset_t oldmask;
	int ret;

	rcu_gc_registry();
	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

#include "urcu-call-rcu-impl.h"
#include "urcu-defer-impl.h"
Commit	Line	Data
fdee2e6d MD	1	/*
	2	* urcu-bp.c
	3	*
	4	* Userspace RCU library, "bulletproof" version.
	5	*
6982d6d7	6	* Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d MD	7	* Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
	8	*
	9	* This library is free software; you can redistribute it and/or
	10	* modify it under the terms of the GNU Lesser General Public
	11	* License as published by the Free Software Foundation; either
	12	* version 2.1 of the License, or (at your option) any later version.
	13	*
	14	* This library is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	17	* Lesser General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU Lesser General Public
	20	* License along with this library; if not, write to the Free Software
	21	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	22	*
	23	* IBM's contributions to this file may be relicensed under LGPLv2 or later.
	24	*/
	25
0617bf4c	26	#define _GNU_SOURCE
fdee2e6d MD	27	#include <stdio.h>
	28	#include <pthread.h>
	29	#include <signal.h>
	30	#include <assert.h>
	31	#include <stdlib.h>
	32	#include <string.h>
	33	#include <errno.h>
	34	#include <poll.h>
	35	#include <unistd.h>
	36	#include <sys/mman.h>
	37
57760d44	38	#include "urcu/map/urcu-bp.h"
5e77fc1f	39
af7c2dbe	40	#include "urcu/static/urcu-bp.h"
fdee2e6d MD	41	/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
	42	#include "urcu-bp.h"
	43
4c1ae2ea MD	44	#ifndef MAP_ANONYMOUS
	45	#define MAP_ANONYMOUS MAP_ANON
	46	#endif
	47
fdee2e6d MD	48	/* Sleep delay in us */
	49	#define RCU_SLEEP_DELAY 1000
	50	#define ARENA_INIT_ALLOC 16
	51
02be5561	52	void __attribute__((destructor)) rcu_bp_exit(void);
fdee2e6d	53
6abb4bd5	54	static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d MD	55
	56	#ifdef DEBUG_YIELD
	57	unsigned int yield_active;
	58	unsigned int __thread rand_yield;
	59	#endif
	60
	61	/*
	62	* Global grace period counter.
02be5561	63	* Contains the current RCU_GP_CTR_PHASE.
fdee2e6d MD	64	* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
	65	* Written to only by writer with mutex taken. Read by both writer and readers.
	66	*/
02be5561	67	long rcu_gp_ctr = RCU_GP_COUNT;
fdee2e6d MD	68
	69	/*
	70	* Pointer to registry elements. Written to only by each individual reader. Read
	71	* by both the reader and the writers.
	72	*/
02be5561	73	struct rcu_reader __thread *rcu_reader;
fdee2e6d	74
16aa9ee8	75	static CDS_LIST_HEAD(registry);
fdee2e6d MD	76
	77	struct registry_arena {
	78	void *p;
	79	size_t len;
	80	size_t used;
	81	};
	82
	83	static struct registry_arena registry_arena;
	84
4cf1675f MD	85	/* Saved fork signal mask, protected by rcu_gp_lock */
	86	static sigset_t saved_fork_signal_mask;
	87
fdee2e6d MD	88	static void rcu_gc_registry(void);
fdee2e6d MD	89
6abb4bd5	90	static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d MD	91	{
	92	int ret;
	93
	94	#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5	95	ret = pthread_mutex_lock(mutex);
fdee2e6d MD	96	if (ret) {
	97	perror("Error in pthread mutex lock");
	98	exit(-1);
	99	}
	100	#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5	101	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
fdee2e6d MD	102	if (ret != EBUSY && ret != EINTR) {
	103	printf("ret = %d, errno = %d\n", ret, errno);
	104	perror("Error in pthread mutex lock");
	105	exit(-1);
	106	}
fdee2e6d MD	107	poll(NULL,0,10);
	108	}
	109	#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
	110	}
	111
6abb4bd5	112	static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d MD	113	{
	114	int ret;
	115
6abb4bd5	116	ret = pthread_mutex_unlock(mutex);
fdee2e6d MD	117	if (ret) {
	118	perror("Error in pthread mutex unlock");
	119	exit(-1);
	120	}
	121	}
	122
2dfb8b5e	123	void update_counter_and_wait(void)
fdee2e6d	124	{
16aa9ee8	125	CDS_LIST_HEAD(qsreaders);
fdee2e6d	126	int wait_loops = 0;
02be5561	127	struct rcu_reader index, tmp;
fdee2e6d	128
32c15e4e	129	/* Switch parity: 0 -> 1, 1 -> 0 */
6cf3827c	130	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
2dfb8b5e MD	131
	132	/*
	133	* Must commit qparity update to memory before waiting for other parity
	134	* quiescent state. Failure to do so could result in the writer waiting
	135	* forever while new readers are always accessing data (no progress).
6cf3827c	136	* Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
2dfb8b5e MD	137	*/
	138
	139	/*
5481ddb3	140	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
2dfb8b5e MD	141	* model easier to understand. It does not have a big performance impact
	142	* anyway, given this is the write-side.
	143	*/
5481ddb3	144	cmm_smp_mb();
2dfb8b5e	145
fdee2e6d	146	/*
02be5561	147	* Wait for each thread rcu_reader.ctr count to become 0.
fdee2e6d MD	148	*/
	149	for (;;) {
	150	wait_loops++;
16aa9ee8	151	cds_list_for_each_entry_safe(index, tmp, &registry, node) {
fdee2e6d	152	if (!rcu_old_gp_ongoing(&index->ctr))
16aa9ee8	153	cds_list_move(&index->node, &qsreaders);
fdee2e6d MD	154	}
fdee2e6d MD	155
16aa9ee8	156	if (cds_list_empty(&registry)) {
fdee2e6d MD	157	break;
	158	} else {
	159	if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
	160	usleep(RCU_SLEEP_DELAY);
	161	else
06f22bdb	162	caa_cpu_relax();
fdee2e6d MD	163	}
	164	}
	165	/* put back the reader list in the registry */
16aa9ee8	166	cds_list_splice(&qsreaders, &registry);
fdee2e6d MD	167	}
	168
	169	void synchronize_rcu(void)
	170	{
	171	sigset_t newmask, oldmask;
	172	int ret;
	173
	174	ret = sigemptyset(&newmask);
	175	assert(!ret);
	176	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	177	assert(!ret);
	178
6abb4bd5	179	mutex_lock(&rcu_gp_lock);
fdee2e6d	180
16aa9ee8	181	if (cds_list_empty(&registry))
2dfb8b5e	182	goto out;
fdee2e6d MD	183
fdee2e6d MD	184	/* All threads should read qparity before accessing data structure
2dfb8b5e	185	* where new ptr points to. */
fdee2e6d	186	/* Write new ptr before changing the qparity */
5481ddb3	187	cmm_smp_mb();
fdee2e6d	188
2dfb8b5e MD	189	/* Remove old registry elements */
2dfb8b5e MD	190	rcu_gc_registry();
fdee2e6d MD	191
	192	/*
	193	* Wait for previous parity to be empty of readers.
	194	*/
2dfb8b5e	195	update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */
fdee2e6d MD	196
fdee2e6d MD	197	/*
5481ddb3	198	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d MD	199	* model easier to understand. It does not have a big performance impact
	200	* anyway, given this is the write-side.
	201	*/
5481ddb3	202	cmm_smp_mb();
fdee2e6d	203
fdee2e6d	204	/*
2dfb8b5e	205	* Wait for previous parity to be empty of readers.
fdee2e6d	206	*/
2dfb8b5e	207	update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */
fdee2e6d MD	208
fdee2e6d MD	209	/*
2dfb8b5e MD	210	* Finish waiting for reader threads before letting the old ptr being
2dfb8b5e MD	211	* freed.
fdee2e6d	212	*/
5481ddb3	213	cmm_smp_mb();
2dfb8b5e	214	out:
6abb4bd5	215	mutex_unlock(&rcu_gp_lock);
fdee2e6d MD	216	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	217	assert(!ret);
	218	}
	219
	220	/*
	221	* library wrappers to be used by non-LGPL compatible source code.
	222	*/
	223
	224	void rcu_read_lock(void)
	225	{
	226	_rcu_read_lock();
	227	}
	228
	229	void rcu_read_unlock(void)
	230	{
	231	_rcu_read_unlock();
	232	}
	233
	234	/*
	235	* only grow for now.
	236	*/
	237	static void resize_arena(struct registry_arena *arena, size_t len)
	238	{
	239	void *new_arena;
	240
0617bf4c MD	241	if (!arena->p)
	242	new_arena = mmap(arena->p, len,
	243	PROT_READ \| PROT_WRITE,
	244	MAP_ANONYMOUS \| MAP_PRIVATE,
	245	-1, 0);
	246	else
	247	new_arena = mremap(arena->p, arena->len,
	248	len, MREMAP_MAYMOVE);
	249	assert(new_arena != MAP_FAILED);
	250
fdee2e6d MD	251	/*
	252	* re-used the same region ?
	253	*/
	254	if (new_arena == arena->p)
	255	return;
	256
fdee2e6d MD	257	bzero(new_arena + arena->len, len - arena->len);
	258	arena->p = new_arena;
	259	}
	260
	261	/* Called with signals off and mutex locked */
	262	static void add_thread(void)
	263	{
02be5561	264	struct rcu_reader *rcu_reader_reg;
fdee2e6d MD	265
fdee2e6d MD	266	if (registry_arena.len
02be5561	267	< registry_arena.used + sizeof(struct rcu_reader))
fdee2e6d MD	268	resize_arena(&registry_arena,
	269	max(registry_arena.len << 1, ARENA_INIT_ALLOC));
	270	/*
	271	* Find a free spot.
	272	*/
02be5561 MD	273	for (rcu_reader_reg = registry_arena.p;
	274	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	275	rcu_reader_reg++) {
	276	if (!rcu_reader_reg->alloc)
fdee2e6d MD	277	break;
fdee2e6d MD	278	}
02be5561 MD	279	rcu_reader_reg->alloc = 1;
02be5561 MD	280	registry_arena.used += sizeof(struct rcu_reader);
fdee2e6d MD	281
fdee2e6d MD	282	/* Add to registry */
02be5561 MD	283	rcu_reader_reg->tid = pthread_self();
02be5561 MD	284	assert(rcu_reader_reg->ctr == 0);
16aa9ee8	285	cds_list_add(&rcu_reader_reg->node, &registry);
02be5561	286	rcu_reader = rcu_reader_reg;
fdee2e6d MD	287	}
	288
	289	/* Called with signals off and mutex locked */
	290	static void rcu_gc_registry(void)
	291	{
02be5561	292	struct rcu_reader *rcu_reader_reg;
fdee2e6d MD	293	pthread_t tid;
	294	int ret;
	295
02be5561 MD	296	for (rcu_reader_reg = registry_arena.p;
	297	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	298	rcu_reader_reg++) {
	299	if (!rcu_reader_reg->alloc)
fdee2e6d	300	continue;
02be5561	301	tid = rcu_reader_reg->tid;
fdee2e6d MD	302	ret = pthread_kill(tid, 0);
	303	assert(ret != EINVAL);
	304	if (ret == ESRCH) {
16aa9ee8	305	cds_list_del(&rcu_reader_reg->node);
79266051	306	rcu_reader_reg->ctr = 0;
02be5561 MD	307	rcu_reader_reg->alloc = 0;
02be5561 MD	308	registry_arena.used -= sizeof(struct rcu_reader);
fdee2e6d MD	309	}
	310	}
	311	}
	312
	313	/* Disable signals, take mutex, add to registry */
	314	void rcu_bp_register(void)
	315	{
	316	sigset_t newmask, oldmask;
	317	int ret;
	318
	319	ret = sigemptyset(&newmask);
	320	assert(!ret);
	321	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	322	assert(!ret);
	323
	324	/*
	325	* Check if a signal concurrently registered our thread since
	326	* the check in rcu_read_lock(). */
02be5561	327	if (rcu_reader)
fdee2e6d MD	328	goto end;
fdee2e6d MD	329
6abb4bd5	330	mutex_lock(&rcu_gp_lock);
fdee2e6d	331	add_thread();
6abb4bd5	332	mutex_unlock(&rcu_gp_lock);
fdee2e6d MD	333	end:
	334	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	335	assert(!ret);
	336	}
	337
02be5561	338	void rcu_bp_exit()
fdee2e6d MD	339	{
	340	munmap(registry_arena.p, registry_arena.len);
	341	}
4cf1675f MD	342
	343	/*
	344	* Holding the rcu_gp_lock across fork will make sure we fork() don't race with
	345	* a concurrent thread executing with this same lock held. This ensures that the
	346	* registry is in a coherent state in the child.
	347	*/
	348	void rcu_bp_before_fork(void)
	349	{
	350	sigset_t newmask, oldmask;
	351	int ret;
	352
	353	ret = sigemptyset(&newmask);
	354	assert(!ret);
	355	ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
	356	assert(!ret);
	357	mutex_lock(&rcu_gp_lock);
	358	saved_fork_signal_mask = oldmask;
	359	}
	360
	361	void rcu_bp_after_fork_parent(void)
	362	{
	363	sigset_t oldmask;
	364	int ret;
	365
	366	oldmask = saved_fork_signal_mask;
	367	mutex_unlock(&rcu_gp_lock);
	368	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	369	assert(!ret);
	370	}
	371
	372	void rcu_bp_after_fork_child(void)
	373	{
	374	sigset_t oldmask;
	375	int ret;
	376
	377	rcu_gc_registry();
	378	oldmask = saved_fork_signal_mask;
	379	mutex_unlock(&rcu_gp_lock);
	380	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	381	assert(!ret);
	382	}
5e77fc1f PM	383
5e77fc1f PM	384	#include "urcu-call-rcu-impl.h"
0376e7b2	385	#include "urcu-defer-impl.h"