[urcu.git] / src / urcu-wait.h

// SPDX-FileCopyrightText: 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
//
// SPDX-License-Identifier: LGPL-2.1-or-later

#ifndef _URCU_WAIT_H
#define _URCU_WAIT_H

/*
 * Userspace RCU library wait/wakeup management
 */

#include <urcu/assert.h>
#include <urcu/uatomic.h>
#include <urcu/wfstack.h>
#include <urcu/futex.h>
#include "urcu-die.h"

/*
 * Number of busy-loop attempts before waiting on futex for grace period
 * batching.
 */
#define URCU_WAIT_ATTEMPTS 1000

enum urcu_wait_state {
	/* URCU_WAIT_WAITING is compared directly (futex compares it). */
	URCU_WAIT_WAITING =	0,
	/* non-zero are used as masks. */
	URCU_WAIT_WAKEUP =	(1 << 0),
	URCU_WAIT_RUNNING =	(1 << 1),
	URCU_WAIT_TEARDOWN =	(1 << 2),
};

struct urcu_wait_node {
	struct cds_wfs_node node;
	int32_t state;	/* enum urcu_wait_state */
};

#define URCU_WAIT_NODE_INIT(name, _state)		\
	{ .node = { .next = NULL }, .state = _state }

#define DEFINE_URCU_WAIT_NODE(name, state)		\
	struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)

#define DECLARE_URCU_WAIT_NODE(name)			\
	struct urcu_wait_node name

struct urcu_wait_queue {
	struct cds_wfs_stack stack;
};

#define URCU_WAIT_QUEUE_HEAD_INIT(name)				\
	{							\
		 .stack = {					\
			.head = CDS_WFS_END,			\
			.lock = PTHREAD_MUTEX_INITIALIZER,	\
		},						\
	}

#define DECLARE_URCU_WAIT_QUEUE(name)			\
	struct urcu_wait_queue name

#define DEFINE_URCU_WAIT_QUEUE(name)			\
	struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name)

struct urcu_waiters {
	struct cds_wfs_head *head;
};

/*
 * Add ourself atomically to a wait queue. Return 0 if queue was
 * previously empty, else return 1.
 * A full memory barrier is issued before being added to the wait queue.
 */
static inline
bool urcu_wait_add(struct urcu_wait_queue *queue,
		struct urcu_wait_node *node)
{
	return cds_wfs_push(&queue->stack, &node->node);
}

/*
 * Atomically move all waiters from wait queue into our local struct
 * urcu_waiters.
 */
static inline
void urcu_move_waiters(struct urcu_waiters *waiters,
		struct urcu_wait_queue *queue)
{
	waiters->head = __cds_wfs_pop_all(&queue->stack);
}

static inline
void urcu_wait_set_state(struct urcu_wait_node *node,
		enum urcu_wait_state state)
{
	node->state = state;
}

static inline
void urcu_wait_node_init(struct urcu_wait_node *node,
		enum urcu_wait_state state)
{
	urcu_wait_set_state(node, state);
	cds_wfs_node_init(&node->node);
}

/*
 * Note: urcu_adaptative_wake_up needs "value" to stay allocated
 * throughout its execution. In this scheme, the waiter owns the node
 * memory, and we only allow it to free this memory when it receives the
 * URCU_WAIT_TEARDOWN flag.
 */
static inline
void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
{
	urcu_posix_assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
	uatomic_store(&wait->state, URCU_WAIT_WAKEUP, CMM_RELEASE);
	if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
		if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
				NULL, NULL, 0) < 0)
			urcu_die(errno);
	}
	/* Allow teardown of struct urcu_wait memory. */
	uatomic_or_mo(&wait->state, URCU_WAIT_TEARDOWN, CMM_RELEASE);
}

/*
 * Caller must initialize "value" to URCU_WAIT_WAITING before passing its
 * memory to waker thread.
 */
static inline
void urcu_adaptative_busy_wait(struct urcu_wait_node *wait)
{
	unsigned int i;

	/* Load and test condition before read state */
	cmm_smp_rmb();
	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
		if (uatomic_load(&wait->state, CMM_ACQUIRE) != URCU_WAIT_WAITING)
			goto skip_futex_wait;
		caa_cpu_relax();
	}
	while (uatomic_load(&wait->state, CMM_ACQUIRE) == URCU_WAIT_WAITING) {
		if (!futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING, NULL, NULL, 0)) {
			/*
			 * Prior queued wakeups queued by unrelated code
			 * using the same address can cause futex wait to
			 * return 0 even through the futex value is still
			 * URCU_WAIT_WAITING (spurious wakeups). Check
			 * the value again in user-space to validate
			 * whether it really differs from
			 * URCU_WAIT_WAITING.
			 */
			continue;
		}
		switch (errno) {
		case EAGAIN:
			/* Value already changed. */
			goto skip_futex_wait;
		case EINTR:
			/* Retry if interrupted by signal. */
			break;	/* Get out of switch. Check again. */
		default:
			/* Unexpected error. */
			urcu_die(errno);
		}
	}
skip_futex_wait:

	/* Tell waker thread than we are running. */
	uatomic_or(&wait->state, URCU_WAIT_RUNNING);

	/*
	 * Wait until waker thread lets us know it's ok to tear down
	 * memory allocated for struct urcu_wait.
	 */
	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
		if (uatomic_load(&wait->state, CMM_RELAXED) & URCU_WAIT_TEARDOWN)
			break;
		caa_cpu_relax();
	}
	while (!(uatomic_load(&wait->state, CMM_ACQUIRE) & URCU_WAIT_TEARDOWN))
		poll(NULL, 0, 10);
	urcu_posix_assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
}

static inline
void urcu_wake_all_waiters(struct urcu_waiters *waiters)
{
	struct cds_wfs_node *iter, *iter_n;

	/* Wake all waiters in our stack head */
	cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) {
		struct urcu_wait_node *wait_node =
			caa_container_of(iter, struct urcu_wait_node, node);

		/* Don't wake already running threads */
		if (uatomic_load(&wait_node->state, CMM_RELAXED) & URCU_WAIT_RUNNING)
			continue;
		urcu_adaptative_wake_up(wait_node);
	}
}

#endif /* _URCU_WAIT_H */
Commit	Line	Data
acdb82a2 MJ	1	// SPDX-FileCopyrightText: 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	2	//
	3	// SPDX-License-Identifier: LGPL-2.1-or-later
	4
cba82d7b MD	5	#ifndef _URCU_WAIT_H
	6	#define _URCU_WAIT_H
	7
	8	/*
cba82d7b	9	* Userspace RCU library wait/wakeup management
cba82d7b MD	10	*/
cba82d7b MD	11
01477510	12	#include <urcu/assert.h>
cba82d7b	13	#include <urcu/uatomic.h>
bf6822a6	14	#include <urcu/wfstack.h>
65eaf4b6	15	#include <urcu/futex.h>
b0a841b4	16	#include "urcu-die.h"
cba82d7b MD	17
	18	/*
	19	* Number of busy-loop attempts before waiting on futex for grace period
	20	* batching.
	21	*/
	22	#define URCU_WAIT_ATTEMPTS 1000
	23
	24	enum urcu_wait_state {
	25	/* URCU_WAIT_WAITING is compared directly (futex compares it). */
	26	URCU_WAIT_WAITING = 0,
	27	/* non-zero are used as masks. */
	28	URCU_WAIT_WAKEUP = (1 << 0),
bf6822a6	29	URCU_WAIT_RUNNING = (1 << 1),
cba82d7b MD	30	URCU_WAIT_TEARDOWN = (1 << 2),
	31	};
	32
bf6822a6 MD	33	struct urcu_wait_node {
	34	struct cds_wfs_node node;
	35	int32_t state; /* enum urcu_wait_state */
cba82d7b MD	36	};
cba82d7b MD	37
bf6822a6	38	#define URCU_WAIT_NODE_INIT(name, _state) \
542fb0dc	39	{ .node = { .next = NULL }, .state = _state }
bf6822a6 MD	40
	41	#define DEFINE_URCU_WAIT_NODE(name, state) \
	42	struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)
	43
	44	#define DECLARE_URCU_WAIT_NODE(name) \
	45	struct urcu_wait_node name
	46
	47	struct urcu_wait_queue {
	48	struct cds_wfs_stack stack;
	49	};
	50
cafe8ce1 MD	51	#define URCU_WAIT_QUEUE_HEAD_INIT(name) \
	52	{ \
	53	.stack = { \
	54	.head = CDS_WFS_END, \
	55	.lock = PTHREAD_MUTEX_INITIALIZER, \
	56	}, \
	57	}
bf6822a6 MD	58
	59	#define DECLARE_URCU_WAIT_QUEUE(name) \
	60	struct urcu_wait_queue name
	61
	62	#define DEFINE_URCU_WAIT_QUEUE(name) \
	63	struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name)
	64
	65	struct urcu_waiters {
	66	struct cds_wfs_head *head;
	67	};
	68
	69	/*
	70	* Add ourself atomically to a wait queue. Return 0 if queue was
	71	* previously empty, else return 1.
	72	* A full memory barrier is issued before being added to the wait queue.
	73	*/
	74	static inline
	75	bool urcu_wait_add(struct urcu_wait_queue *queue,
	76	struct urcu_wait_node *node)
	77	{
	78	return cds_wfs_push(&queue->stack, &node->node);
	79	}
	80
	81	/*
	82	* Atomically move all waiters from wait queue into our local struct
	83	* urcu_waiters.
	84	*/
	85	static inline
	86	void urcu_move_waiters(struct urcu_waiters *waiters,
	87	struct urcu_wait_queue *queue)
	88	{
	89	waiters->head = __cds_wfs_pop_all(&queue->stack);
	90	}
	91
	92	static inline
	93	void urcu_wait_set_state(struct urcu_wait_node *node,
	94	enum urcu_wait_state state)
	95	{
	96	node->state = state;
	97	}
	98
cba82d7b	99	static inline
bf6822a6 MD	100	void urcu_wait_node_init(struct urcu_wait_node *node,
bf6822a6 MD	101	enum urcu_wait_state state)
cba82d7b	102	{
bf6822a6 MD	103	urcu_wait_set_state(node, state);
bf6822a6 MD	104	cds_wfs_node_init(&node->node);
cba82d7b MD	105	}
	106
	107	/*
	108	* Note: urcu_adaptative_wake_up needs "value" to stay allocated
bf6822a6	109	* throughout its execution. In this scheme, the waiter owns the node
cba82d7b MD	110	* memory, and we only allow it to free this memory when it receives the
	111	* URCU_WAIT_TEARDOWN flag.
	112	*/
	113	static inline
bf6822a6	114	void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
cba82d7b	115	{
01477510	116	urcu_posix_assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
601922a8	117	uatomic_store(&wait->state, URCU_WAIT_WAKEUP, CMM_RELEASE);
b0a841b4 MD	118	if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
	119	if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
	120	NULL, NULL, 0) < 0)
	121	urcu_die(errno);
	122	}
cba82d7b	123	/* Allow teardown of struct urcu_wait memory. */
2ea38794	124	uatomic_or_mo(&wait->state, URCU_WAIT_TEARDOWN, CMM_RELEASE);
cba82d7b MD	125	}
	126
	127	/*
	128	* Caller must initialize "value" to URCU_WAIT_WAITING before passing its
	129	* memory to waker thread.
	130	*/
bf6822a6 MD	131	static inline
bf6822a6 MD	132	void urcu_adaptative_busy_wait(struct urcu_wait_node *wait)
cba82d7b MD	133	{
	134	unsigned int i;
	135
bf6822a6	136	/* Load and test condition before read state */
cba82d7b MD	137	cmm_smp_rmb();
cba82d7b MD	138	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
601922a8	139	if (uatomic_load(&wait->state, CMM_ACQUIRE) != URCU_WAIT_WAITING)
cba82d7b MD	140	goto skip_futex_wait;
	141	caa_cpu_relax();
	142	}
601922a8	143	while (uatomic_load(&wait->state, CMM_ACQUIRE) == URCU_WAIT_WAITING) {
71693131 MD	144	if (!futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING, NULL, NULL, 0)) {
	145	/*
	146	* Prior queued wakeups queued by unrelated code
	147	* using the same address can cause futex wait to
	148	* return 0 even through the futex value is still
	149	* URCU_WAIT_WAITING (spurious wakeups). Check
	150	* the value again in user-space to validate
	151	* whether it really differs from
	152	* URCU_WAIT_WAITING.
	153	*/
	154	continue;
	155	}
b0a841b4	156	switch (errno) {
71693131	157	case EAGAIN:
b0a841b4 MD	158	/* Value already changed. */
	159	goto skip_futex_wait;
	160	case EINTR:
	161	/* Retry if interrupted by signal. */
71693131	162	break; /* Get out of switch. Check again. */
b0a841b4 MD	163	default:
	164	/* Unexpected error. */
	165	urcu_die(errno);
	166	}
	167	}
cba82d7b MD	168	skip_futex_wait:
cba82d7b MD	169
ffa11a18	170	/* Tell waker thread than we are running. */
bf6822a6	171	uatomic_or(&wait->state, URCU_WAIT_RUNNING);
cba82d7b MD	172
	173	/*
	174	* Wait until waker thread lets us know it's ok to tear down
	175	* memory allocated for struct urcu_wait.
	176	*/
	177	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
601922a8	178	if (uatomic_load(&wait->state, CMM_RELAXED) & URCU_WAIT_TEARDOWN)
cba82d7b MD	179	break;
	180	caa_cpu_relax();
	181	}
2ea38794	182	while (!(uatomic_load(&wait->state, CMM_ACQUIRE) & URCU_WAIT_TEARDOWN))
cba82d7b	183	poll(NULL, 0, 10);
01477510	184	urcu_posix_assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
bf6822a6 MD	185	}
	186
	187	static inline
	188	void urcu_wake_all_waiters(struct urcu_waiters *waiters)
	189	{
	190	struct cds_wfs_node iter, iter_n;
	191
	192	/* Wake all waiters in our stack head */
	193	cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) {
	194	struct urcu_wait_node *wait_node =
	195	caa_container_of(iter, struct urcu_wait_node, node);
	196
	197	/* Don't wake already running threads */
2ea38794	198	if (uatomic_load(&wait_node->state, CMM_RELAXED) & URCU_WAIT_RUNNING)
bf6822a6 MD	199	continue;
	200	urcu_adaptative_wake_up(wait_node);
	201	}
cba82d7b MD	202	}
	203
	204	#endif /* _URCU_WAIT_H */