Commit | Line | Data |
---|---|---|
acdb82a2 MJ |
1 | // SPDX-FileCopyrightText: 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
2 | // | |
3 | // SPDX-License-Identifier: LGPL-2.1-or-later | |
4 | ||
cba82d7b MD |
5 | #ifndef _URCU_WAIT_H |
6 | #define _URCU_WAIT_H | |
7 | ||
8 | /* | |
cba82d7b | 9 | * Userspace RCU library wait/wakeup management |
cba82d7b MD |
10 | */ |
11 | ||
01477510 | 12 | #include <urcu/assert.h> |
cba82d7b | 13 | #include <urcu/uatomic.h> |
bf6822a6 | 14 | #include <urcu/wfstack.h> |
65eaf4b6 | 15 | #include <urcu/futex.h> |
b0a841b4 | 16 | #include "urcu-die.h" |
cba82d7b MD |
17 | |
18 | /* | |
19 | * Number of busy-loop attempts before waiting on futex for grace period | |
20 | * batching. | |
21 | */ | |
22 | #define URCU_WAIT_ATTEMPTS 1000 | |
23 | ||
24 | enum urcu_wait_state { | |
25 | /* URCU_WAIT_WAITING is compared directly (futex compares it). */ | |
26 | URCU_WAIT_WAITING = 0, | |
27 | /* non-zero are used as masks. */ | |
28 | URCU_WAIT_WAKEUP = (1 << 0), | |
bf6822a6 | 29 | URCU_WAIT_RUNNING = (1 << 1), |
cba82d7b MD |
30 | URCU_WAIT_TEARDOWN = (1 << 2), |
31 | }; | |
32 | ||
bf6822a6 MD |
33 | struct urcu_wait_node { |
34 | struct cds_wfs_node node; | |
35 | int32_t state; /* enum urcu_wait_state */ | |
cba82d7b MD |
36 | }; |
37 | ||
bf6822a6 | 38 | #define URCU_WAIT_NODE_INIT(name, _state) \ |
542fb0dc | 39 | { .node = { .next = NULL }, .state = _state } |
bf6822a6 MD |
40 | |
41 | #define DEFINE_URCU_WAIT_NODE(name, state) \ | |
42 | struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state) | |
43 | ||
44 | #define DECLARE_URCU_WAIT_NODE(name) \ | |
45 | struct urcu_wait_node name | |
46 | ||
47 | struct urcu_wait_queue { | |
48 | struct cds_wfs_stack stack; | |
49 | }; | |
50 | ||
cafe8ce1 MD |
51 | #define URCU_WAIT_QUEUE_HEAD_INIT(name) \ |
52 | { \ | |
53 | .stack = { \ | |
54 | .head = CDS_WFS_END, \ | |
55 | .lock = PTHREAD_MUTEX_INITIALIZER, \ | |
56 | }, \ | |
57 | } | |
bf6822a6 MD |
58 | |
59 | #define DECLARE_URCU_WAIT_QUEUE(name) \ | |
60 | struct urcu_wait_queue name | |
61 | ||
62 | #define DEFINE_URCU_WAIT_QUEUE(name) \ | |
63 | struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name) | |
64 | ||
65 | struct urcu_waiters { | |
66 | struct cds_wfs_head *head; | |
67 | }; | |
68 | ||
69 | /* | |
70 | * Add ourself atomically to a wait queue. Return 0 if queue was | |
71 | * previously empty, else return 1. | |
72 | * A full memory barrier is issued before being added to the wait queue. | |
73 | */ | |
74 | static inline | |
75 | bool urcu_wait_add(struct urcu_wait_queue *queue, | |
76 | struct urcu_wait_node *node) | |
77 | { | |
78 | return cds_wfs_push(&queue->stack, &node->node); | |
79 | } | |
80 | ||
81 | /* | |
82 | * Atomically move all waiters from wait queue into our local struct | |
83 | * urcu_waiters. | |
84 | */ | |
85 | static inline | |
86 | void urcu_move_waiters(struct urcu_waiters *waiters, | |
87 | struct urcu_wait_queue *queue) | |
88 | { | |
89 | waiters->head = __cds_wfs_pop_all(&queue->stack); | |
90 | } | |
91 | ||
92 | static inline | |
93 | void urcu_wait_set_state(struct urcu_wait_node *node, | |
94 | enum urcu_wait_state state) | |
95 | { | |
96 | node->state = state; | |
97 | } | |
98 | ||
cba82d7b | 99 | static inline |
bf6822a6 MD |
100 | void urcu_wait_node_init(struct urcu_wait_node *node, |
101 | enum urcu_wait_state state) | |
cba82d7b | 102 | { |
bf6822a6 MD |
103 | urcu_wait_set_state(node, state); |
104 | cds_wfs_node_init(&node->node); | |
cba82d7b MD |
105 | } |
106 | ||
107 | /* | |
108 | * Note: urcu_adaptative_wake_up needs "value" to stay allocated | |
bf6822a6 | 109 | * throughout its execution. In this scheme, the waiter owns the node |
cba82d7b MD |
110 | * memory, and we only allow it to free this memory when it receives the |
111 | * URCU_WAIT_TEARDOWN flag. | |
112 | */ | |
113 | static inline | |
bf6822a6 | 114 | void urcu_adaptative_wake_up(struct urcu_wait_node *wait) |
cba82d7b | 115 | { |
01477510 | 116 | urcu_posix_assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING); |
601922a8 | 117 | uatomic_store(&wait->state, URCU_WAIT_WAKEUP, CMM_RELEASE); |
b0a841b4 MD |
118 | if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) { |
119 | if (futex_noasync(&wait->state, FUTEX_WAKE, 1, | |
120 | NULL, NULL, 0) < 0) | |
121 | urcu_die(errno); | |
122 | } | |
cba82d7b | 123 | /* Allow teardown of struct urcu_wait memory. */ |
2ea38794 | 124 | uatomic_or_mo(&wait->state, URCU_WAIT_TEARDOWN, CMM_RELEASE); |
cba82d7b MD |
125 | } |
126 | ||
127 | /* | |
128 | * Caller must initialize "value" to URCU_WAIT_WAITING before passing its | |
129 | * memory to waker thread. | |
130 | */ | |
bf6822a6 MD |
131 | static inline |
132 | void urcu_adaptative_busy_wait(struct urcu_wait_node *wait) | |
cba82d7b MD |
133 | { |
134 | unsigned int i; | |
135 | ||
bf6822a6 | 136 | /* Load and test condition before read state */ |
cba82d7b MD |
137 | cmm_smp_rmb(); |
138 | for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) { | |
601922a8 | 139 | if (uatomic_load(&wait->state, CMM_ACQUIRE) != URCU_WAIT_WAITING) |
cba82d7b MD |
140 | goto skip_futex_wait; |
141 | caa_cpu_relax(); | |
142 | } | |
601922a8 | 143 | while (uatomic_load(&wait->state, CMM_ACQUIRE) == URCU_WAIT_WAITING) { |
71693131 MD |
144 | if (!futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING, NULL, NULL, 0)) { |
145 | /* | |
146 | * Prior queued wakeups queued by unrelated code | |
147 | * using the same address can cause futex wait to | |
148 | * return 0 even through the futex value is still | |
149 | * URCU_WAIT_WAITING (spurious wakeups). Check | |
150 | * the value again in user-space to validate | |
151 | * whether it really differs from | |
152 | * URCU_WAIT_WAITING. | |
153 | */ | |
154 | continue; | |
155 | } | |
b0a841b4 | 156 | switch (errno) { |
71693131 | 157 | case EAGAIN: |
b0a841b4 MD |
158 | /* Value already changed. */ |
159 | goto skip_futex_wait; | |
160 | case EINTR: | |
161 | /* Retry if interrupted by signal. */ | |
71693131 | 162 | break; /* Get out of switch. Check again. */ |
b0a841b4 MD |
163 | default: |
164 | /* Unexpected error. */ | |
165 | urcu_die(errno); | |
166 | } | |
167 | } | |
cba82d7b MD |
168 | skip_futex_wait: |
169 | ||
ffa11a18 | 170 | /* Tell waker thread than we are running. */ |
bf6822a6 | 171 | uatomic_or(&wait->state, URCU_WAIT_RUNNING); |
cba82d7b MD |
172 | |
173 | /* | |
174 | * Wait until waker thread lets us know it's ok to tear down | |
175 | * memory allocated for struct urcu_wait. | |
176 | */ | |
177 | for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) { | |
601922a8 | 178 | if (uatomic_load(&wait->state, CMM_RELAXED) & URCU_WAIT_TEARDOWN) |
cba82d7b MD |
179 | break; |
180 | caa_cpu_relax(); | |
181 | } | |
2ea38794 | 182 | while (!(uatomic_load(&wait->state, CMM_ACQUIRE) & URCU_WAIT_TEARDOWN)) |
cba82d7b | 183 | poll(NULL, 0, 10); |
01477510 | 184 | urcu_posix_assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN); |
bf6822a6 MD |
185 | } |
186 | ||
187 | static inline | |
188 | void urcu_wake_all_waiters(struct urcu_waiters *waiters) | |
189 | { | |
190 | struct cds_wfs_node *iter, *iter_n; | |
191 | ||
192 | /* Wake all waiters in our stack head */ | |
193 | cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) { | |
194 | struct urcu_wait_node *wait_node = | |
195 | caa_container_of(iter, struct urcu_wait_node, node); | |
196 | ||
197 | /* Don't wake already running threads */ | |
2ea38794 | 198 | if (uatomic_load(&wait_node->state, CMM_RELAXED) & URCU_WAIT_RUNNING) |
bf6822a6 MD |
199 | continue; |
200 | urcu_adaptative_wake_up(wait_node); | |
201 | } | |
cba82d7b MD |
202 | } |
203 | ||
204 | #endif /* _URCU_WAIT_H */ |