Fix rcu_assign_pointer() dynamic linking behavior
[urcu.git] / urcu.c
1 /*
2 * urcu.c
3 *
4 * Userspace RCU library
5 *
6 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
7 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 *
23 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
24 */
25
26 #include <stdio.h>
27 #include <pthread.h>
28 #include <signal.h>
29 #include <assert.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <poll.h>
34
35 #include "urcu-static.h"
36 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
37 #include "urcu.h"
38
39 #ifndef URCU_MB
40 static int init_done;
41
42 void __attribute__((constructor)) urcu_init(void);
43 void __attribute__((destructor)) urcu_exit(void);
44 #else
45 void urcu_init(void)
46 {
47 }
48 #endif
49
50 static pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
51
52 int gp_futex;
53
54 /*
55 * Global grace period counter.
56 * Contains the current RCU_GP_CTR_BIT.
57 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
58 * Written to only by writer with mutex taken. Read by both writer and readers.
59 */
60 long urcu_gp_ctr = RCU_GP_COUNT;
61
62 /*
63 * Written to only by each individual reader. Read by both the reader and the
64 * writers.
65 */
66 struct urcu_reader __thread urcu_reader;
67
68 #ifdef DEBUG_YIELD
69 unsigned int yield_active;
70 unsigned int __thread rand_yield;
71 #endif
72
73 static LIST_HEAD(registry);
74
75 static void internal_urcu_lock(void)
76 {
77 int ret;
78
79 #ifndef DISTRUST_SIGNALS_EXTREME
80 ret = pthread_mutex_lock(&urcu_mutex);
81 if (ret) {
82 perror("Error in pthread mutex lock");
83 exit(-1);
84 }
85 #else /* #ifndef DISTRUST_SIGNALS_EXTREME */
86 while ((ret = pthread_mutex_trylock(&urcu_mutex)) != 0) {
87 if (ret != EBUSY && ret != EINTR) {
88 printf("ret = %d, errno = %d\n", ret, errno);
89 perror("Error in pthread mutex lock");
90 exit(-1);
91 }
92 if (urcu_reader.need_mb) {
93 smp_mb();
94 urcu_reader.need_mb = 0;
95 smp_mb();
96 }
97 poll(NULL,0,10);
98 }
99 #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
100 }
101
102 static void internal_urcu_unlock(void)
103 {
104 int ret;
105
106 ret = pthread_mutex_unlock(&urcu_mutex);
107 if (ret) {
108 perror("Error in pthread mutex unlock");
109 exit(-1);
110 }
111 }
112
113 /*
114 * called with urcu_mutex held.
115 */
116 static void switch_next_urcu_qparity(void)
117 {
118 STORE_SHARED(urcu_gp_ctr, urcu_gp_ctr ^ RCU_GP_CTR_BIT);
119 }
120
121 #ifdef URCU_MB
122 #if 0 /* unused */
123 static void force_mb_single_thread(struct urcu_reader *index)
124 {
125 smp_mb();
126 }
127 #endif //0
128
129 static void force_mb_all_threads(void)
130 {
131 smp_mb();
132 }
133 #else /* #ifdef URCU_MB */
134 #if 0 /* unused */
135 static void force_mb_single_thread(struct urcu_reader *index)
136 {
137 assert(!list_empty(&registry));
138 /*
139 * pthread_kill has a smp_mb(). But beware, we assume it performs
140 * a cache flush on architectures with non-coherent cache. Let's play
141 * safe and don't assume anything : we use smp_mc() to make sure the
142 * cache flush is enforced.
143 */
144 index->need_mb = 1;
145 smp_mc(); /* write ->need_mb before sending the signals */
146 pthread_kill(index->tid, SIGURCU);
147 smp_mb();
148 /*
149 * Wait for sighandler (and thus mb()) to execute on every thread.
150 * BUSY-LOOP.
151 */
152 while (index->need_mb) {
153 poll(NULL, 0, 1);
154 }
155 smp_mb(); /* read ->need_mb before ending the barrier */
156 }
157 #endif //0
158
159 static void force_mb_all_threads(void)
160 {
161 struct urcu_reader *index;
162
163 /*
164 * Ask for each threads to execute a smp_mb() so we can consider the
165 * compiler barriers around rcu read lock as real memory barriers.
166 */
167 if (list_empty(&registry))
168 return;
169 /*
170 * pthread_kill has a smp_mb(). But beware, we assume it performs
171 * a cache flush on architectures with non-coherent cache. Let's play
172 * safe and don't assume anything : we use smp_mc() to make sure the
173 * cache flush is enforced.
174 */
175 list_for_each_entry(index, &registry, head) {
176 index->need_mb = 1;
177 smp_mc(); /* write need_mb before sending the signal */
178 pthread_kill(index->tid, SIGURCU);
179 }
180 /*
181 * Wait for sighandler (and thus mb()) to execute on every thread.
182 *
183 * Note that the pthread_kill() will never be executed on systems
184 * that correctly deliver signals in a timely manner. However, it
185 * is not uncommon for kernels to have bugs that can result in
186 * lost or unduly delayed signals.
187 *
188 * If you are seeing the below pthread_kill() executing much at
189 * all, we suggest testing the underlying kernel and filing the
190 * relevant bug report. For Linux kernels, we recommend getting
191 * the Linux Test Project (LTP).
192 */
193 list_for_each_entry(index, &registry, head) {
194 while (index->need_mb) {
195 pthread_kill(index->tid, SIGURCU);
196 poll(NULL, 0, 1);
197 }
198 }
199 smp_mb(); /* read ->need_mb before ending the barrier */
200 }
201 #endif /* #else #ifdef URCU_MB */
202
203 /*
204 * synchronize_rcu() waiting. Single thread.
205 */
206 static void wait_gp(void)
207 {
208 /* Read reader_gp before read futex */
209 force_mb_all_threads();
210 if (uatomic_read(&gp_futex) == -1)
211 futex(&gp_futex, FUTEX_WAIT, -1,
212 NULL, NULL, 0);
213 }
214
215 void wait_for_quiescent_state(void)
216 {
217 LIST_HEAD(qsreaders);
218 int wait_loops = 0;
219 struct urcu_reader *index, *tmp;
220
221 if (list_empty(&registry))
222 return;
223 /*
224 * Wait for each thread urcu_reader.ctr count to become 0.
225 */
226 for (;;) {
227 wait_loops++;
228 if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
229 uatomic_dec(&gp_futex);
230 /* Write futex before read reader_gp */
231 force_mb_all_threads();
232 }
233
234 list_for_each_entry_safe(index, tmp, &registry, head) {
235 if (!rcu_old_gp_ongoing(&index->ctr))
236 list_move(&index->head, &qsreaders);
237 }
238
239 #ifndef HAS_INCOHERENT_CACHES
240 if (list_empty(&registry)) {
241 if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
242 /* Read reader_gp before write futex */
243 force_mb_all_threads();
244 uatomic_set(&gp_futex, 0);
245 }
246 break;
247 } else {
248 if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
249 wait_gp();
250 else
251 cpu_relax();
252 }
253 #else /* #ifndef HAS_INCOHERENT_CACHES */
254 /*
255 * BUSY-LOOP. Force the reader thread to commit its
256 * urcu_reader.ctr update to memory if we wait for too long.
257 */
258 if (list_empty(&registry)) {
259 if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
260 /* Read reader_gp before write futex */
261 force_mb_all_threads();
262 uatomic_set(&gp_futex, 0);
263 }
264 break;
265 } else {
266 switch (wait_loops) {
267 case RCU_QS_ACTIVE_ATTEMPTS:
268 wait_gp();
269 break; /* only escape switch */
270 case KICK_READER_LOOPS:
271 force_mb_all_threads();
272 wait_loops = 0;
273 break; /* only escape switch */
274 default:
275 cpu_relax();
276 }
277 }
278 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
279 }
280 /* put back the reader list in the registry */
281 list_splice(&qsreaders, &registry);
282 }
283
284 void synchronize_rcu(void)
285 {
286 internal_urcu_lock();
287
288 /* All threads should read qparity before accessing data structure
289 * where new ptr points to. Must be done within internal_urcu_lock
290 * because it iterates on reader threads.*/
291 /* Write new ptr before changing the qparity */
292 force_mb_all_threads();
293
294 switch_next_urcu_qparity(); /* 0 -> 1 */
295
296 /*
297 * Must commit qparity update to memory before waiting for parity
298 * 0 quiescent state. Failure to do so could result in the writer
299 * waiting forever while new readers are always accessing data (no
300 * progress).
301 * Ensured by STORE_SHARED and LOAD_SHARED.
302 */
303
304 /*
305 * Adding a smp_mb() which is _not_ formally required, but makes the
306 * model easier to understand. It does not have a big performance impact
307 * anyway, given this is the write-side.
308 */
309 smp_mb();
310
311 /*
312 * Wait for previous parity to be empty of readers.
313 */
314 wait_for_quiescent_state(); /* Wait readers in parity 0 */
315
316 /*
317 * Must finish waiting for quiescent state for parity 0 before
318 * committing qparity update to memory. Failure to do so could result in
319 * the writer waiting forever while new readers are always accessing
320 * data (no progress).
321 * Ensured by STORE_SHARED and LOAD_SHARED.
322 */
323
324 /*
325 * Adding a smp_mb() which is _not_ formally required, but makes the
326 * model easier to understand. It does not have a big performance impact
327 * anyway, given this is the write-side.
328 */
329 smp_mb();
330
331 switch_next_urcu_qparity(); /* 1 -> 0 */
332
333 /*
334 * Must commit qparity update to memory before waiting for parity
335 * 1 quiescent state. Failure to do so could result in the writer
336 * waiting forever while new readers are always accessing data (no
337 * progress).
338 * Ensured by STORE_SHARED and LOAD_SHARED.
339 */
340
341 /*
342 * Adding a smp_mb() which is _not_ formally required, but makes the
343 * model easier to understand. It does not have a big performance impact
344 * anyway, given this is the write-side.
345 */
346 smp_mb();
347
348 /*
349 * Wait for previous parity to be empty of readers.
350 */
351 wait_for_quiescent_state(); /* Wait readers in parity 1 */
352
353 /* Finish waiting for reader threads before letting the old ptr being
354 * freed. Must be done within internal_urcu_lock because it iterates on
355 * reader threads. */
356 force_mb_all_threads();
357
358 internal_urcu_unlock();
359 }
360
361 /*
362 * library wrappers to be used by non-LGPL compatible source code.
363 */
364
365 void rcu_read_lock(void)
366 {
367 _rcu_read_lock();
368 }
369
370 void rcu_read_unlock(void)
371 {
372 _rcu_read_unlock();
373 }
374
375 void *rcu_dereference(void *p)
376 {
377 return _rcu_dereference(p);
378 }
379
380 void *rcu_assign_pointer_sym(void **p, void *v)
381 {
382 wmb();
383 return STORE_SHARED(*p, v);
384 }
385
386 void *rcu_xchg_pointer_sym(void **p, void *v)
387 {
388 wmb();
389 return uatomic_xchg(p, v);
390 }
391
392 void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new)
393 {
394 wmb();
395 return uatomic_cmpxchg(p, old, _new);
396 }
397
398 void *rcu_publish_content_sym(void **p, void *v)
399 {
400 void *oldptr;
401
402 oldptr = _rcu_xchg_pointer(p, v);
403 synchronize_rcu();
404 return oldptr;
405 }
406
407 void rcu_register_thread(void)
408 {
409 urcu_reader.tid = pthread_self();
410 assert(urcu_reader.need_mb == 0);
411 assert(urcu_reader.ctr == 0);
412
413 internal_urcu_lock();
414 urcu_init(); /* In case gcc does not support constructor attribute */
415 list_add(&urcu_reader.head, &registry);
416 internal_urcu_unlock();
417 }
418
419 void rcu_unregister_thread(void)
420 {
421 internal_urcu_lock();
422 list_del(&urcu_reader.head);
423 internal_urcu_unlock();
424 }
425
426 #ifndef URCU_MB
427 static void sigurcu_handler(int signo, siginfo_t *siginfo, void *context)
428 {
429 /*
430 * Executing this smp_mb() is the only purpose of this signal handler.
431 * It punctually promotes barrier() into smp_mb() on every thread it is
432 * executed on.
433 */
434 smp_mb();
435 urcu_reader.need_mb = 0;
436 smp_mb();
437 }
438
439 /*
440 * urcu_init constructor. Called when the library is linked, but also when
441 * reader threads are calling rcu_register_thread().
442 * Should only be called by a single thread at a given time. This is ensured by
443 * holing the internal_urcu_lock() from rcu_register_thread() or by running at
444 * library load time, which should not be executed by multiple threads nor
445 * concurrently with rcu_register_thread() anyway.
446 */
447 void urcu_init(void)
448 {
449 struct sigaction act;
450 int ret;
451
452 if (init_done)
453 return;
454 init_done = 1;
455
456 act.sa_sigaction = sigurcu_handler;
457 act.sa_flags = SA_SIGINFO | SA_RESTART;
458 sigemptyset(&act.sa_mask);
459 ret = sigaction(SIGURCU, &act, NULL);
460 if (ret) {
461 perror("Error in sigaction");
462 exit(-1);
463 }
464 }
465
466 void urcu_exit(void)
467 {
468 struct sigaction act;
469 int ret;
470
471 ret = sigaction(SIGURCU, NULL, &act);
472 if (ret) {
473 perror("Error in sigaction");
474 exit(-1);
475 }
476 assert(act.sa_sigaction == sigurcu_handler);
477 assert(list_empty(&registry));
478 }
479 #endif /* #ifndef URCU_MB */
This page took 0.057671 seconds and 4 git commands to generate.