Revert "Use initial-exec tls model"
[urcu.git] / src / urcu-bp.c
CommitLineData
fdee2e6d
MD
1/*
2 * urcu-bp.c
3 *
4 * Userspace RCU library, "bulletproof" version.
5 *
6982d6d7 6 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d
MD
7 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 *
23 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
24 */
25
71c811bf 26#define _LGPL_SOURCE
fdee2e6d
MD
27#include <stdio.h>
28#include <pthread.h>
29#include <signal.h>
30#include <assert.h>
31#include <stdlib.h>
32#include <string.h>
33#include <errno.h>
34#include <poll.h>
35#include <unistd.h>
36#include <sys/mman.h>
37
999991c6 38#include "urcu/arch.h"
d73fb81f 39#include "urcu/wfcqueue.h"
57760d44 40#include "urcu/map/urcu-bp.h"
af7c2dbe 41#include "urcu/static/urcu-bp.h"
618b2595 42#include "urcu-pointer.h"
bd252a04 43#include "urcu/tls-compat.h"
71c811bf 44
4a6d7378
MD
45#include "urcu-die.h"
46
fdee2e6d 47/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
71c811bf 48#undef _LGPL_SOURCE
fdee2e6d 49#include "urcu-bp.h"
71c811bf 50#define _LGPL_SOURCE
fdee2e6d 51
4c1ae2ea
MD
52#ifndef MAP_ANONYMOUS
53#define MAP_ANONYMOUS MAP_ANON
54#endif
55
c7eaf61c
MD
56#ifdef __linux__
57static
58void *mremap_wrapper(void *old_address, size_t old_size,
59 size_t new_size, int flags)
60{
61 return mremap(old_address, old_size, new_size, flags);
62}
63#else
45a4872f
MD
64
65#define MREMAP_MAYMOVE 1
66#define MREMAP_FIXED 2
67
68/*
95b94246 69 * mremap wrapper for non-Linux systems not allowing MAYMOVE.
45a4872f
MD
70 * This is not generic.
71*/
c7eaf61c
MD
72static
73void *mremap_wrapper(void *old_address, size_t old_size,
74 size_t new_size, int flags)
45a4872f 75{
95b94246
MD
76 assert(!(flags & MREMAP_MAYMOVE));
77
78 return MAP_FAILED;
45a4872f
MD
79}
80#endif
81
9340c38d
MD
82/* Sleep delay in ms */
83#define RCU_SLEEP_DELAY_MS 10
95b94246
MD
84#define INIT_NR_THREADS 8
85#define ARENA_INIT_ALLOC \
86 sizeof(struct registry_chunk) \
87 + INIT_NR_THREADS * sizeof(struct rcu_reader)
fdee2e6d 88
b7b6a8f5
PB
89/*
90 * Active attempts to check for reader Q.S. before calling sleep().
91 */
92#define RCU_QS_ACTIVE_ATTEMPTS 100
93
76d6a951
MD
94static
95int rcu_bp_refcount;
96
999991c6
MD
97/* If the headers do not support membarrier system call, fall back smp_mb. */
98#ifdef __NR_membarrier
99# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
f541831e
MD
100#else
101# define membarrier(...) -ENOSYS
102#endif
103
104enum membarrier_cmd {
105 MEMBARRIER_CMD_QUERY = 0,
106 MEMBARRIER_CMD_SHARED = (1 << 0),
107};
108
c1be8fb9
MD
109static
110void __attribute__((constructor)) rcu_bp_init(void);
111static
02be5561 112void __attribute__((destructor)) rcu_bp_exit(void);
fdee2e6d 113
d8d9a340 114#ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
f541831e 115int urcu_bp_has_sys_membarrier;
d8d9a340 116#endif
f541831e 117
731ccb96
MD
118/*
119 * rcu_gp_lock ensures mutual exclusion between threads calling
120 * synchronize_rcu().
121 */
6abb4bd5 122static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
731ccb96
MD
123/*
124 * rcu_registry_lock ensures mutual exclusion between threads
125 * registering and unregistering themselves to/from the registry, and
126 * with threads reading that registry from synchronize_rcu(). However,
127 * this lock is not held all the way through the completion of awaiting
128 * for the grace period. It is sporadically released between iterations
129 * on the registry.
130 * rcu_registry_lock may nest inside rcu_gp_lock.
131 */
132static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d 133
c1be8fb9
MD
134static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
135static int initialized;
136
137static pthread_key_t urcu_bp_key;
138
c13c2e55 139struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
fdee2e6d
MD
140
141/*
142 * Pointer to registry elements. Written to only by each individual reader. Read
143 * by both the reader and the writers.
144 */
2f661865 145DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);
fdee2e6d 146
16aa9ee8 147static CDS_LIST_HEAD(registry);
fdee2e6d 148
95b94246
MD
149struct registry_chunk {
150 size_t data_len; /* data length */
c1be8fb9 151 size_t used; /* amount of data used */
95b94246
MD
152 struct cds_list_head node; /* chunk_list node */
153 char data[];
154};
155
fdee2e6d 156struct registry_arena {
95b94246 157 struct cds_list_head chunk_list;
fdee2e6d
MD
158};
159
95b94246
MD
160static struct registry_arena registry_arena = {
161 .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
162};
fdee2e6d 163
4cf1675f
MD
164/* Saved fork signal mask, protected by rcu_gp_lock */
165static sigset_t saved_fork_signal_mask;
166
6abb4bd5 167static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d
MD
168{
169 int ret;
170
171#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5 172 ret = pthread_mutex_lock(mutex);
4a6d7378
MD
173 if (ret)
174 urcu_die(ret);
fdee2e6d 175#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5 176 while ((ret = pthread_mutex_trylock(mutex)) != 0) {
4a6d7378
MD
177 if (ret != EBUSY && ret != EINTR)
178 urcu_die(ret);
fdee2e6d
MD
179 poll(NULL,0,10);
180 }
181#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
182}
183
6abb4bd5 184static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d
MD
185{
186 int ret;
187
6abb4bd5 188 ret = pthread_mutex_unlock(mutex);
4a6d7378
MD
189 if (ret)
190 urcu_die(ret);
fdee2e6d
MD
191}
192
f541831e
MD
193static void smp_mb_master(void)
194{
195 if (caa_likely(urcu_bp_has_sys_membarrier))
196 (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
197 else
198 cmm_smp_mb();
199}
200
731ccb96
MD
201/*
202 * Always called with rcu_registry lock held. Releases this lock between
203 * iterations and grabs it again. Holds the lock when it returns.
204 */
52c75091
MD
205static void wait_for_readers(struct cds_list_head *input_readers,
206 struct cds_list_head *cur_snap_readers,
207 struct cds_list_head *qsreaders)
fdee2e6d 208{
9340c38d 209 unsigned int wait_loops = 0;
02be5561 210 struct rcu_reader *index, *tmp;
fdee2e6d 211
fdee2e6d 212 /*
dd61d077
MD
213 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
214 * indicate quiescence (not nested), or observe the current
c13c2e55 215 * rcu_gp.ctr value.
fdee2e6d
MD
216 */
217 for (;;) {
9340c38d
MD
218 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
219 wait_loops++;
220
52c75091
MD
221 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
222 switch (rcu_reader_state(&index->ctr)) {
223 case RCU_READER_ACTIVE_CURRENT:
224 if (cur_snap_readers) {
225 cds_list_move(&index->node,
226 cur_snap_readers);
227 break;
228 }
229 /* Fall-through */
230 case RCU_READER_INACTIVE:
231 cds_list_move(&index->node, qsreaders);
232 break;
233 case RCU_READER_ACTIVE_OLD:
234 /*
235 * Old snapshot. Leaving node in
236 * input_readers will make us busy-loop
237 * until the snapshot becomes current or
238 * the reader becomes inactive.
239 */
240 break;
241 }
fdee2e6d
MD
242 }
243
52c75091 244 if (cds_list_empty(input_readers)) {
fdee2e6d
MD
245 break;
246 } else {
731ccb96
MD
247 /* Temporarily unlock the registry lock. */
248 mutex_unlock(&rcu_registry_lock);
9340c38d
MD
249 if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
250 (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
fdee2e6d 251 else
06f22bdb 252 caa_cpu_relax();
731ccb96
MD
253 /* Re-lock the registry lock before the next loop. */
254 mutex_lock(&rcu_registry_lock);
fdee2e6d
MD
255 }
256 }
fdee2e6d
MD
257}
258
259void synchronize_rcu(void)
260{
52c75091
MD
261 CDS_LIST_HEAD(cur_snap_readers);
262 CDS_LIST_HEAD(qsreaders);
fdee2e6d
MD
263 sigset_t newmask, oldmask;
264 int ret;
265
6ed4b2e6 266 ret = sigfillset(&newmask);
fdee2e6d 267 assert(!ret);
6ed4b2e6 268 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
fdee2e6d
MD
269 assert(!ret);
270
6abb4bd5 271 mutex_lock(&rcu_gp_lock);
fdee2e6d 272
731ccb96
MD
273 mutex_lock(&rcu_registry_lock);
274
16aa9ee8 275 if (cds_list_empty(&registry))
2dfb8b5e 276 goto out;
fdee2e6d
MD
277
278 /* All threads should read qparity before accessing data structure
2dfb8b5e 279 * where new ptr points to. */
fdee2e6d 280 /* Write new ptr before changing the qparity */
f541831e 281 smp_mb_master();
fdee2e6d 282
fdee2e6d 283 /*
dd61d077 284 * Wait for readers to observe original parity or be quiescent.
731ccb96
MD
285 * wait_for_readers() can release and grab again rcu_registry_lock
286 * interally.
dd61d077 287 */
52c75091 288 wait_for_readers(&registry, &cur_snap_readers, &qsreaders);
dd61d077
MD
289
290 /*
291 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
292 * model easier to understand. It does not have a big performance impact
293 * anyway, given this is the write-side.
294 */
295 cmm_smp_mb();
296
297 /* Switch parity: 0 -> 1, 1 -> 0 */
c13c2e55 298 CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
dd61d077
MD
299
300 /*
301 * Must commit qparity update to memory before waiting for other parity
302 * quiescent state. Failure to do so could result in the writer waiting
303 * forever while new readers are always accessing data (no progress).
304 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
fdee2e6d 305 */
fdee2e6d
MD
306
307 /*
5481ddb3 308 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d
MD
309 * model easier to understand. It does not have a big performance impact
310 * anyway, given this is the write-side.
311 */
5481ddb3 312 cmm_smp_mb();
fdee2e6d 313
fdee2e6d 314 /*
dd61d077 315 * Wait for readers to observe new parity or be quiescent.
731ccb96
MD
316 * wait_for_readers() can release and grab again rcu_registry_lock
317 * interally.
fdee2e6d 318 */
52c75091
MD
319 wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
320
321 /*
322 * Put quiescent reader list back into registry.
323 */
324 cds_list_splice(&qsreaders, &registry);
fdee2e6d
MD
325
326 /*
2dfb8b5e
MD
327 * Finish waiting for reader threads before letting the old ptr being
328 * freed.
fdee2e6d 329 */
f541831e 330 smp_mb_master();
2dfb8b5e 331out:
731ccb96 332 mutex_unlock(&rcu_registry_lock);
6abb4bd5 333 mutex_unlock(&rcu_gp_lock);
fdee2e6d
MD
334 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
335 assert(!ret);
336}
337
338/*
339 * library wrappers to be used by non-LGPL compatible source code.
340 */
341
342void rcu_read_lock(void)
343{
344 _rcu_read_lock();
345}
346
347void rcu_read_unlock(void)
348{
349 _rcu_read_unlock();
350}
351
882f3357
MD
352int rcu_read_ongoing(void)
353{
354 return _rcu_read_ongoing();
355}
356
fdee2e6d 357/*
95b94246
MD
358 * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
359 * Else, try expanding the last chunk. If this fails, allocate a new
360 * chunk twice as big as the last chunk.
361 * Memory used by chunks _never_ moves. A chunk could theoretically be
362 * freed when all "used" slots are released, but we don't do it at this
363 * point.
fdee2e6d 364 */
95b94246
MD
365static
366void expand_arena(struct registry_arena *arena)
fdee2e6d 367{
95b94246
MD
368 struct registry_chunk *new_chunk, *last_chunk;
369 size_t old_chunk_len, new_chunk_len;
370
371 /* No chunk. */
372 if (cds_list_empty(&arena->chunk_list)) {
373 assert(ARENA_INIT_ALLOC >=
374 sizeof(struct registry_chunk)
375 + sizeof(struct rcu_reader));
376 new_chunk_len = ARENA_INIT_ALLOC;
5592d049
MJ
377 new_chunk = (struct registry_chunk *) mmap(NULL,
378 new_chunk_len,
9d8612b7
MD
379 PROT_READ | PROT_WRITE,
380 MAP_ANONYMOUS | MAP_PRIVATE,
381 -1, 0);
95b94246
MD
382 if (new_chunk == MAP_FAILED)
383 abort();
d3ac5bb7 384 memset(new_chunk, 0, new_chunk_len);
95b94246
MD
385 new_chunk->data_len =
386 new_chunk_len - sizeof(struct registry_chunk);
387 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
388 return; /* We're done. */
389 }
9d8612b7 390
95b94246
MD
391 /* Try expanding last chunk. */
392 last_chunk = cds_list_entry(arena->chunk_list.prev,
393 struct registry_chunk, node);
394 old_chunk_len =
395 last_chunk->data_len + sizeof(struct registry_chunk);
396 new_chunk_len = old_chunk_len << 1;
397
398 /* Don't allow memory mapping to move, just expand. */
399 new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
400 new_chunk_len, 0);
401 if (new_chunk != MAP_FAILED) {
402 /* Should not have moved. */
403 assert(new_chunk == last_chunk);
d3ac5bb7 404 memset((char *) last_chunk + old_chunk_len, 0,
95b94246
MD
405 new_chunk_len - old_chunk_len);
406 last_chunk->data_len =
407 new_chunk_len - sizeof(struct registry_chunk);
408 return; /* We're done. */
409 }
0617bf4c 410
95b94246 411 /* Remap did not succeed, we need to add a new chunk. */
5592d049
MJ
412 new_chunk = (struct registry_chunk *) mmap(NULL,
413 new_chunk_len,
95b94246
MD
414 PROT_READ | PROT_WRITE,
415 MAP_ANONYMOUS | MAP_PRIVATE,
416 -1, 0);
417 if (new_chunk == MAP_FAILED)
418 abort();
d3ac5bb7 419 memset(new_chunk, 0, new_chunk_len);
95b94246
MD
420 new_chunk->data_len =
421 new_chunk_len - sizeof(struct registry_chunk);
422 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
423}
fdee2e6d 424
95b94246
MD
425static
426struct rcu_reader *arena_alloc(struct registry_arena *arena)
427{
428 struct registry_chunk *chunk;
429 struct rcu_reader *rcu_reader_reg;
430 int expand_done = 0; /* Only allow to expand once per alloc */
431 size_t len = sizeof(struct rcu_reader);
432
433retry:
434 cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
435 if (chunk->data_len - chunk->used < len)
436 continue;
437 /* Find spot */
438 for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
439 rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
440 rcu_reader_reg++) {
441 if (!rcu_reader_reg->alloc) {
442 rcu_reader_reg->alloc = 1;
443 chunk->used += len;
444 return rcu_reader_reg;
445 }
446 }
447 }
448
449 if (!expand_done) {
450 expand_arena(arena);
451 expand_done = 1;
452 goto retry;
453 }
454
455 return NULL;
fdee2e6d
MD
456}
457
458/* Called with signals off and mutex locked */
95b94246
MD
459static
460void add_thread(void)
fdee2e6d 461{
02be5561 462 struct rcu_reader *rcu_reader_reg;
c1be8fb9 463 int ret;
fdee2e6d 464
95b94246
MD
465 rcu_reader_reg = arena_alloc(&registry_arena);
466 if (!rcu_reader_reg)
467 abort();
c1be8fb9
MD
468 ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
469 if (ret)
470 abort();
fdee2e6d
MD
471
472 /* Add to registry */
02be5561
MD
473 rcu_reader_reg->tid = pthread_self();
474 assert(rcu_reader_reg->ctr == 0);
16aa9ee8 475 cds_list_add(&rcu_reader_reg->node, &registry);
95b94246
MD
476 /*
477 * Reader threads are pointing to the reader registry. This is
478 * why its memory should never be relocated.
479 */
bd252a04 480 URCU_TLS(rcu_reader) = rcu_reader_reg;
fdee2e6d
MD
481}
482
c1be8fb9
MD
483/* Called with mutex locked */
484static
485void cleanup_thread(struct registry_chunk *chunk,
486 struct rcu_reader *rcu_reader_reg)
487{
488 rcu_reader_reg->ctr = 0;
489 cds_list_del(&rcu_reader_reg->node);
490 rcu_reader_reg->tid = 0;
491 rcu_reader_reg->alloc = 0;
492 chunk->used -= sizeof(struct rcu_reader);
493}
494
495static
496struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
fdee2e6d 497{
95b94246 498 struct registry_chunk *chunk;
fdee2e6d 499
95b94246 500 cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
c1be8fb9
MD
501 if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
502 continue;
503 if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
504 continue;
505 return chunk;
506 }
507 return NULL;
508}
95b94246 509
c1be8fb9
MD
510/* Called with signals off and mutex locked */
511static
76d6a951 512void remove_thread(struct rcu_reader *rcu_reader_reg)
c1be8fb9 513{
c1be8fb9
MD
514 cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
515 URCU_TLS(rcu_reader) = NULL;
fdee2e6d
MD
516}
517
518/* Disable signals, take mutex, add to registry */
519void rcu_bp_register(void)
520{
521 sigset_t newmask, oldmask;
522 int ret;
523
6ed4b2e6 524 ret = sigfillset(&newmask);
c1be8fb9
MD
525 if (ret)
526 abort();
6ed4b2e6 527 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
c1be8fb9
MD
528 if (ret)
529 abort();
fdee2e6d
MD
530
531 /*
532 * Check if a signal concurrently registered our thread since
c1be8fb9
MD
533 * the check in rcu_read_lock().
534 */
bd252a04 535 if (URCU_TLS(rcu_reader))
fdee2e6d
MD
536 goto end;
537
c1be8fb9
MD
538 /*
539 * Take care of early registration before urcu_bp constructor.
540 */
541 rcu_bp_init();
542
731ccb96 543 mutex_lock(&rcu_registry_lock);
fdee2e6d 544 add_thread();
731ccb96 545 mutex_unlock(&rcu_registry_lock);
fdee2e6d
MD
546end:
547 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
c1be8fb9
MD
548 if (ret)
549 abort();
550}
551
552/* Disable signals, take mutex, remove from registry */
553static
76d6a951 554void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
c1be8fb9
MD
555{
556 sigset_t newmask, oldmask;
557 int ret;
558
559 ret = sigfillset(&newmask);
560 if (ret)
561 abort();
562 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
563 if (ret)
564 abort();
565
731ccb96 566 mutex_lock(&rcu_registry_lock);
76d6a951 567 remove_thread(rcu_reader_reg);
731ccb96 568 mutex_unlock(&rcu_registry_lock);
c1be8fb9
MD
569 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
570 if (ret)
571 abort();
76d6a951 572 rcu_bp_exit();
c1be8fb9
MD
573}
574
575/*
576 * Remove thread from the registry when it exits, and flag it as
577 * destroyed so garbage collection can take care of it.
578 */
579static
580void urcu_bp_thread_exit_notifier(void *rcu_key)
581{
76d6a951 582 rcu_bp_unregister(rcu_key);
c1be8fb9
MD
583}
584
d8d9a340
MD
585#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
586static
587void rcu_sys_membarrier_status(int available)
588{
589 if (!available)
590 abort();
591}
592#else
593static
594void rcu_sys_membarrier_status(int available)
595{
64478021
MD
596 /*
597 * membarrier has blocking behavior, which changes the
598 * application behavior too much compared to using barriers when
599 * synchronize_rcu is used repeatedly (without using call_rcu).
600 * Don't use membarrier for now, unless its use has been
601 * explicitly forced when building liburcu.
602 */
d8d9a340
MD
603}
604#endif
605
c1be8fb9
MD
606static
607void rcu_bp_init(void)
608{
609 mutex_lock(&init_lock);
76d6a951 610 if (!rcu_bp_refcount++) {
c1be8fb9
MD
611 int ret;
612
613 ret = pthread_key_create(&urcu_bp_key,
614 urcu_bp_thread_exit_notifier);
615 if (ret)
616 abort();
f541831e 617 ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
d8d9a340
MD
618 rcu_sys_membarrier_status(ret >= 0
619 && (ret & MEMBARRIER_CMD_SHARED));
c1be8fb9
MD
620 initialized = 1;
621 }
622 mutex_unlock(&init_lock);
fdee2e6d
MD
623}
624
c1be8fb9 625static
9380711a 626void rcu_bp_exit(void)
fdee2e6d 627{
76d6a951
MD
628 mutex_lock(&init_lock);
629 if (!--rcu_bp_refcount) {
630 struct registry_chunk *chunk, *tmp;
631 int ret;
95b94246 632
76d6a951
MD
633 cds_list_for_each_entry_safe(chunk, tmp,
634 &registry_arena.chunk_list, node) {
5592d049 635 munmap((void *) chunk, chunk->data_len
76d6a951
MD
636 + sizeof(struct registry_chunk));
637 }
7937ae1c 638 CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);
76d6a951
MD
639 ret = pthread_key_delete(urcu_bp_key);
640 if (ret)
641 abort();
95b94246 642 }
76d6a951 643 mutex_unlock(&init_lock);
fdee2e6d 644}
4cf1675f
MD
645
646/*
731ccb96
MD
647 * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
648 * sure we fork() don't race with a concurrent thread executing with
649 * any of those locks held. This ensures that the registry and data
650 * protected by rcu_gp_lock are in a coherent state in the child.
4cf1675f
MD
651 */
652void rcu_bp_before_fork(void)
653{
654 sigset_t newmask, oldmask;
655 int ret;
656
6ed4b2e6 657 ret = sigfillset(&newmask);
4cf1675f 658 assert(!ret);
6ed4b2e6 659 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
4cf1675f
MD
660 assert(!ret);
661 mutex_lock(&rcu_gp_lock);
731ccb96 662 mutex_lock(&rcu_registry_lock);
4cf1675f
MD
663 saved_fork_signal_mask = oldmask;
664}
665
666void rcu_bp_after_fork_parent(void)
667{
668 sigset_t oldmask;
669 int ret;
670
671 oldmask = saved_fork_signal_mask;
731ccb96 672 mutex_unlock(&rcu_registry_lock);
4cf1675f
MD
673 mutex_unlock(&rcu_gp_lock);
674 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
675 assert(!ret);
676}
677
c1be8fb9
MD
678/*
679 * Prune all entries from registry except our own thread. Fits the Linux
731ccb96 680 * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
c1be8fb9
MD
681 */
682static
683void urcu_bp_prune_registry(void)
684{
685 struct registry_chunk *chunk;
686 struct rcu_reader *rcu_reader_reg;
687
688 cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
689 for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
690 rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
691 rcu_reader_reg++) {
692 if (!rcu_reader_reg->alloc)
693 continue;
694 if (rcu_reader_reg->tid == pthread_self())
695 continue;
696 cleanup_thread(chunk, rcu_reader_reg);
697 }
698 }
699}
700
4cf1675f
MD
701void rcu_bp_after_fork_child(void)
702{
703 sigset_t oldmask;
704 int ret;
705
c1be8fb9 706 urcu_bp_prune_registry();
4cf1675f 707 oldmask = saved_fork_signal_mask;
731ccb96 708 mutex_unlock(&rcu_registry_lock);
4cf1675f
MD
709 mutex_unlock(&rcu_gp_lock);
710 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
711 assert(!ret);
712}
5e77fc1f 713
9b7981bb
MD
714void *rcu_dereference_sym_bp(void *p)
715{
716 return _rcu_dereference(p);
717}
718
5efd3cd2
MD
719void *rcu_set_pointer_sym_bp(void **p, void *v)
720{
721 cmm_wmb();
424d4ed5
MD
722 uatomic_set(p, v);
723 return v;
5efd3cd2
MD
724}
725
726void *rcu_xchg_pointer_sym_bp(void **p, void *v)
727{
728 cmm_wmb();
729 return uatomic_xchg(p, v);
730}
731
732void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new)
733{
734 cmm_wmb();
735 return uatomic_cmpxchg(p, old, _new);
736}
737
5e6b23a6 738DEFINE_RCU_FLAVOR(rcu_flavor);
541d828d 739
5e77fc1f 740#include "urcu-call-rcu-impl.h"
0376e7b2 741#include "urcu-defer-impl.h"
This page took 0.117553 seconds and 4 git commands to generate.