src/urcu-bp.c

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 /*
   7  * Userspace RCU library, "bulletproof" version.
   8  *
   9  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  10  */
  11
  12 #define URCU_NO_COMPAT_IDENTIFIERS
  13 #define _LGPL_SOURCE
  14 #include <stdio.h>
  15 #include <pthread.h>
  16 #include <signal.h>
  17 #include <stdlib.h>
  18 #include <string.h>
  19 #include <errno.h>
  20 #include <poll.h>
  21 #include <unistd.h>
  22 #include <stdbool.h>
  23 #include <sys/mman.h>
  24
  25 #include <urcu/annotate.h>
  26 #include <urcu/assert.h>
  27 #include <urcu/config.h>
  28 #include <urcu/arch.h>
  29 #include <urcu/wfcqueue.h>
  30 #include <urcu/map/urcu-bp.h>
  31 #include <urcu/static/urcu-bp.h>
  32 #include <urcu/pointer.h>
  33 #include <urcu/tls-compat.h>
  34
  35 #include "urcu-die.h"
  36 #include "urcu-utils.h"
  37
  38 #define URCU_API_MAP
  39 /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
  40 #undef _LGPL_SOURCE
  41 #include <urcu/urcu-bp.h>
  42 #define _LGPL_SOURCE
  43
  44 #ifndef MAP_ANONYMOUS
  45 #define MAP_ANONYMOUS MAP_ANON
  46 #endif
  47
  48 #ifdef __linux__
  49 static
  50 void *mremap_wrapper(void *old_address, size_t old_size,
  51                 size_t new_size, int flags)
  52 {
  53         return mremap(old_address, old_size, new_size, flags);
  54 }
  55 #else
  56
  57 #define MREMAP_MAYMOVE  1
  58 #define MREMAP_FIXED    2
  59
  60 /*
  61  * mremap wrapper for non-Linux systems not allowing MAYMOVE.
  62  * This is not generic.
  63 */
  64 static
  65 void *mremap_wrapper(void *old_address __attribute__((unused)),
  66                 size_t old_size __attribute__((unused)),
  67                 size_t new_size __attribute__((unused)),
  68                 int flags)
  69 {
  70         urcu_posix_assert(!(flags & MREMAP_MAYMOVE));
  71
  72         return MAP_FAILED;
  73 }
  74 #endif
  75
  76 /* Sleep delay in ms */
  77 #define RCU_SLEEP_DELAY_MS      10
  78 #define INIT_READER_COUNT       8
  79
  80 /*
  81  * Active attempts to check for reader Q.S. before calling sleep().
  82  */
  83 #define RCU_QS_ACTIVE_ATTEMPTS 100
  84
  85 static
  86 int urcu_bp_refcount;
  87
  88 /* If the headers do not support membarrier system call, fall back smp_mb. */
  89 #ifdef __NR_membarrier
  90 # define membarrier(...)                syscall(__NR_membarrier, __VA_ARGS__)
  91 #else
  92 # define membarrier(...)                -ENOSYS
  93 #endif
  94
  95 enum membarrier_cmd {
  96         MEMBARRIER_CMD_QUERY                            = 0,
  97         MEMBARRIER_CMD_SHARED                           = (1 << 0),
  98         /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
  99         /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
 100         MEMBARRIER_CMD_PRIVATE_EXPEDITED                = (1 << 3),
 101         MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED       = (1 << 4),
 102 };
 103
 104 static
 105 void __attribute__((constructor)) _urcu_bp_init(void);
 106 static
 107 void urcu_bp_exit(void);
 108 static
 109 void __attribute__((destructor)) urcu_bp_exit_destructor(void);
 110 static void urcu_call_rcu_exit(void);
 111
 112 #ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 113 int urcu_bp_has_sys_membarrier;
 114 #endif
 115
 116 /*
 117  * rcu_gp_lock ensures mutual exclusion between threads calling
 118  * synchronize_rcu().
 119  */
 120 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
 121 /*
 122  * rcu_registry_lock ensures mutual exclusion between threads
 123  * registering and unregistering themselves to/from the registry, and
 124  * with threads reading that registry from synchronize_rcu(). However,
 125  * this lock is not held all the way through the completion of awaiting
 126  * for the grace period. It is sporadically released between iterations
 127  * on the registry.
 128  * rcu_registry_lock may nest inside rcu_gp_lock.
 129  */
 130 static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
 131
 132 static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
 133 static int initialized;
 134
 135 static pthread_key_t urcu_bp_key;
 136
 137 struct urcu_bp_gp urcu_bp_gp = { .ctr = URCU_BP_GP_COUNT };
 138
 139 /*
 140  * Pointer to registry elements. Written to only by each individual reader. Read
 141  * by both the reader and the writers.
 142  */
 143 DEFINE_URCU_TLS(struct urcu_bp_reader *, urcu_bp_reader);
 144
 145 static CDS_LIST_HEAD(registry);
 146
 147 struct registry_chunk {
 148         size_t capacity;                /* capacity of this chunk (in elements) */
 149         size_t used;                    /* count of elements used */
 150         struct cds_list_head node;      /* chunk_list node */
 151         struct urcu_bp_reader readers[];
 152 };
 153
 154 struct registry_arena {
 155         struct cds_list_head chunk_list;
 156 };
 157
 158 static struct registry_arena registry_arena = {
 159         .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
 160 };
 161
 162 /* Saved fork signal mask, protected by rcu_gp_lock */
 163 static sigset_t saved_fork_signal_mask;
 164
 165 static void mutex_lock(pthread_mutex_t *mutex)
 166 {
 167         int ret;
 168
 169 #ifndef DISTRUST_SIGNALS_EXTREME
 170         ret = pthread_mutex_lock(mutex);
 171         if (ret)
 172                 urcu_die(ret);
 173 #else /* #ifndef DISTRUST_SIGNALS_EXTREME */
 174         while ((ret = pthread_mutex_trylock(mutex)) != 0) {
 175                 if (ret != EBUSY && ret != EINTR)
 176                         urcu_die(ret);
 177                 poll(NULL,0,10);
 178         }
 179 #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
 180 }
 181
 182 static void mutex_unlock(pthread_mutex_t *mutex)
 183 {
 184         int ret;
 185
 186         ret = pthread_mutex_unlock(mutex);
 187         if (ret)
 188                 urcu_die(ret);
 189 }
 190
 191 static void smp_mb_master(void)
 192 {
 193         if (caa_likely(urcu_bp_has_sys_membarrier)) {
 194                 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0))
 195                         urcu_die(errno);
 196         } else {
 197                 cmm_smp_mb();
 198         }
 199 }
 200
 201 /* Get the size of a chunk's allocation from its capacity (an element count). */
 202 static size_t chunk_allocation_size(size_t capacity)
 203 {
 204         return (capacity * sizeof(struct urcu_bp_reader)) +
 205                 sizeof(struct registry_chunk);
 206 }
 207
 208 /*
 209  * Always called with rcu_registry lock held. Releases this lock between
 210  * iterations and grabs it again. Holds the lock when it returns.
 211  */
 212 static void wait_for_readers(struct cds_list_head *input_readers,
 213                         struct cds_list_head *cur_snap_readers,
 214                         struct cds_list_head *qsreaders,
 215                         cmm_annotate_t *group)
 216 {
 217         unsigned int wait_loops = 0;
 218         struct urcu_bp_reader *index, *tmp;
 219
 220         /*
 221          * Wait for each thread URCU_TLS(urcu_bp_reader).ctr to either
 222          * indicate quiescence (not nested), or observe the current
 223          * rcu_gp.ctr value.
 224          */
 225         for (;;) {
 226                 if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
 227                         wait_loops++;
 228
 229                 cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
 230                         switch (urcu_bp_reader_state(&index->ctr, group)) {
 231                         case URCU_BP_READER_ACTIVE_CURRENT:
 232                                 if (cur_snap_readers) {
 233                                         cds_list_move(&index->node,
 234                                                 cur_snap_readers);
 235                                         break;
 236                                 }
 237                                 /* Fall-through */
 238                         case URCU_BP_READER_INACTIVE:
 239                                 cds_list_move(&index->node, qsreaders);
 240                                 break;
 241                         case URCU_BP_READER_ACTIVE_OLD:
 242                                 /*
 243                                  * Old snapshot. Leaving node in
 244                                  * input_readers will make us busy-loop
 245                                  * until the snapshot becomes current or
 246                                  * the reader becomes inactive.
 247                                  */
 248                                 break;
 249                         }
 250                 }
 251
 252                 if (cds_list_empty(input_readers)) {
 253                         break;
 254                 } else {
 255                         /* Temporarily unlock the registry lock. */
 256                         mutex_unlock(&rcu_registry_lock);
 257                         if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
 258                                 (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
 259                         else
 260                                 caa_cpu_relax();
 261                         /* Re-lock the registry lock before the next loop. */
 262                         mutex_lock(&rcu_registry_lock);
 263                 }
 264         }
 265 }
 266
 267 void urcu_bp_synchronize_rcu(void)
 268 {
 269         cmm_annotate_define(acquire_group);
 270         cmm_annotate_define(release_group);
 271         CDS_LIST_HEAD(cur_snap_readers);
 272         CDS_LIST_HEAD(qsreaders);
 273         sigset_t newmask, oldmask;
 274         int ret;
 275
 276         ret = sigfillset(&newmask);
 277         urcu_posix_assert(!ret);
 278         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 279         urcu_posix_assert(!ret);
 280
 281         mutex_lock(&rcu_gp_lock);
 282
 283         mutex_lock(&rcu_registry_lock);
 284
 285         if (cds_list_empty(&registry))
 286                 goto out;
 287
 288         /* All threads should read qparity before accessing data structure
 289          * where new ptr points to. */
 290         /* Write new ptr before changing the qparity */
 291         smp_mb_master();
 292         cmm_annotate_group_mb_release(&release_group);
 293
 294         /*
 295          * Wait for readers to observe original parity or be quiescent.
 296          * wait_for_readers() can release and grab again rcu_registry_lock
 297          * internally.
 298          */
 299         wait_for_readers(&registry, &cur_snap_readers, &qsreaders, &acquire_group);
 300
 301         /*
 302          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 303          * model easier to understand. It does not have a big performance impact
 304          * anyway, given this is the write-side.
 305          */
 306         cmm_smp_mb();
 307
 308         /* Switch parity: 0 -> 1, 1 -> 0 */
 309         cmm_annotate_group_mem_release(&release_group, &rcu_gp.ctr);
 310         uatomic_store(&rcu_gp.ctr, rcu_gp.ctr ^ URCU_BP_GP_CTR_PHASE, CMM_RELAXED);
 311
 312         /*
 313          * Must commit qparity update to memory before waiting for other parity
 314          * quiescent state. Failure to do so could result in the writer waiting
 315          * forever while new readers are always accessing data (no progress).
 316          * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
 317          */
 318
 319         /*
 320          * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
 321          * model easier to understand. It does not have a big performance impact
 322          * anyway, given this is the write-side.
 323          */
 324         cmm_smp_mb();
 325
 326         /*
 327          * Wait for readers to observe new parity or be quiescent.
 328          * wait_for_readers() can release and grab again rcu_registry_lock
 329          * internally.
 330          */
 331         wait_for_readers(&cur_snap_readers, NULL, &qsreaders, &acquire_group);
 332
 333         /*
 334          * Put quiescent reader list back into registry.
 335          */
 336         cds_list_splice(&qsreaders, &registry);
 337
 338         /*
 339          * Finish waiting for reader threads before letting the old ptr being
 340          * freed.
 341          */
 342         smp_mb_master();
 343         cmm_annotate_group_mb_acquire(&acquire_group);
 344 out:
 345         mutex_unlock(&rcu_registry_lock);
 346         mutex_unlock(&rcu_gp_lock);
 347         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 348         urcu_posix_assert(!ret);
 349 }
 350
 351 /*
 352  * library wrappers to be used by non-LGPL compatible source code.
 353  */
 354
 355 void urcu_bp_read_lock(void)
 356 {
 357         _urcu_bp_read_lock();
 358 }
 359
 360 void urcu_bp_read_unlock(void)
 361 {
 362         _urcu_bp_read_unlock();
 363 }
 364
 365 int urcu_bp_read_ongoing(void)
 366 {
 367         return _urcu_bp_read_ongoing();
 368 }
 369
 370 /*
 371  * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
 372  * Else, try expanding the last chunk. If this fails, allocate a new
 373  * chunk twice as big as the last chunk.
 374  * Memory used by chunks _never_ moves. A chunk could theoretically be
 375  * freed when all "used" slots are released, but we don't do it at this
 376  * point.
 377  */
 378 static
 379 void expand_arena(struct registry_arena *arena)
 380 {
 381         struct registry_chunk *new_chunk, *last_chunk;
 382         size_t old_chunk_size_bytes, new_chunk_size_bytes, new_capacity;
 383
 384         /* No chunk. */
 385         if (cds_list_empty(&arena->chunk_list)) {
 386                 new_chunk_size_bytes = chunk_allocation_size(INIT_READER_COUNT);
 387                 new_chunk = (struct registry_chunk *) mmap(NULL,
 388                         new_chunk_size_bytes,
 389                         PROT_READ | PROT_WRITE,
 390                         MAP_ANONYMOUS | MAP_PRIVATE,
 391                         -1, 0);
 392                 if (new_chunk == MAP_FAILED)
 393                         abort();
 394                 memset(new_chunk, 0, new_chunk_size_bytes);
 395                 new_chunk->capacity = INIT_READER_COUNT;
 396                 cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
 397                 return;         /* We're done. */
 398         }
 399
 400         /* Try expanding last chunk. */
 401         last_chunk = cds_list_entry(arena->chunk_list.prev,
 402                 struct registry_chunk, node);
 403         old_chunk_size_bytes = chunk_allocation_size(last_chunk->capacity);
 404         new_capacity = last_chunk->capacity << 1;
 405         new_chunk_size_bytes = chunk_allocation_size(new_capacity);
 406
 407         /* Don't allow memory mapping to move, just expand. */
 408         new_chunk = mremap_wrapper(last_chunk, old_chunk_size_bytes,
 409                 new_chunk_size_bytes, 0);
 410         if (new_chunk != MAP_FAILED) {
 411                 /* Should not have moved. */
 412                 assert(new_chunk == last_chunk);
 413                 memset((char *) last_chunk + old_chunk_size_bytes, 0,
 414                         new_chunk_size_bytes - old_chunk_size_bytes);
 415                 last_chunk->capacity = new_capacity;
 416                 return;         /* We're done. */
 417         }
 418
 419         /* Remap did not succeed, we need to add a new chunk. */
 420         new_chunk = (struct registry_chunk *) mmap(NULL,
 421                 new_chunk_size_bytes,
 422                 PROT_READ | PROT_WRITE,
 423                 MAP_ANONYMOUS | MAP_PRIVATE,
 424                 -1, 0);
 425         if (new_chunk == MAP_FAILED)
 426                 abort();
 427         memset(new_chunk, 0, new_chunk_size_bytes);
 428         new_chunk->capacity = new_capacity;
 429         cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
 430 }
 431
 432 static
 433 struct rcu_reader *arena_alloc(struct registry_arena *arena)
 434 {
 435         struct registry_chunk *chunk;
 436         int expand_done = 0;    /* Only allow to expand once per alloc */
 437
 438 retry:
 439         cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
 440                 size_t spot_idx;
 441
 442                 /* Skip fully used chunks. */
 443                 if (chunk->used == chunk->capacity) {
 444                         continue;
 445                 }
 446
 447                 /* Find a spot. */
 448                 for (spot_idx = 0; spot_idx < chunk->capacity; spot_idx++) {
 449                         if (!chunk->readers[spot_idx].alloc) {
 450                                 chunk->readers[spot_idx].alloc = 1;
 451                                 chunk->used++;
 452                                 return &chunk->readers[spot_idx];
 453                         }
 454                 }
 455         }
 456
 457         if (!expand_done) {
 458                 expand_arena(arena);
 459                 expand_done = 1;
 460                 goto retry;
 461         }
 462
 463         return NULL;
 464 }
 465
 466 /* Called with signals off and mutex locked */
 467 static
 468 void add_thread(void)
 469 {
 470         struct rcu_reader *rcu_reader_reg;
 471         int ret;
 472
 473         rcu_reader_reg = arena_alloc(&registry_arena);
 474         if (!rcu_reader_reg)
 475                 abort();
 476         ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
 477         if (ret)
 478                 abort();
 479
 480         /* Add to registry */
 481         rcu_reader_reg->tid = pthread_self();
 482         urcu_posix_assert(rcu_reader_reg->ctr == 0);
 483         cds_list_add(&rcu_reader_reg->node, &registry);
 484         /*
 485          * Reader threads are pointing to the reader registry. This is
 486          * why its memory should never be relocated.
 487          */
 488         URCU_TLS(urcu_bp_reader) = rcu_reader_reg;
 489 }
 490
 491 /* Called with mutex locked */
 492 static
 493 void cleanup_thread(struct registry_chunk *chunk,
 494                 struct rcu_reader *rcu_reader_reg)
 495 {
 496         rcu_reader_reg->ctr = 0;
 497         cds_list_del(&rcu_reader_reg->node);
 498         rcu_reader_reg->tid = 0;
 499         rcu_reader_reg->alloc = 0;
 500         chunk->used--;
 501 }
 502
 503 static
 504 struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
 505 {
 506         struct registry_chunk *chunk;
 507
 508         cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
 509                 if (rcu_reader_reg < (struct urcu_bp_reader *) &chunk->readers[0])
 510                         continue;
 511                 if (rcu_reader_reg >= (struct urcu_bp_reader *) &chunk->readers[chunk->capacity])
 512                         continue;
 513                 return chunk;
 514         }
 515         return NULL;
 516 }
 517
 518 /* Called with signals off and mutex locked */
 519 static
 520 void remove_thread(struct rcu_reader *rcu_reader_reg)
 521 {
 522         cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
 523         URCU_TLS(urcu_bp_reader) = NULL;
 524 }
 525
 526 /* Disable signals, take mutex, add to registry */
 527 void urcu_bp_register(void)
 528 {
 529         sigset_t newmask, oldmask;
 530         int ret;
 531
 532         ret = sigfillset(&newmask);
 533         if (ret)
 534                 abort();
 535         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 536         if (ret)
 537                 abort();
 538
 539         /*
 540          * Check if a signal concurrently registered our thread since
 541          * the check in rcu_read_lock().
 542          */
 543         if (URCU_TLS(urcu_bp_reader))
 544                 goto end;
 545
 546         /*
 547          * Take care of early registration before urcu_bp constructor.
 548          */
 549         _urcu_bp_init();
 550
 551         mutex_lock(&rcu_registry_lock);
 552         add_thread();
 553         mutex_unlock(&rcu_registry_lock);
 554 end:
 555         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 556         if (ret)
 557                 abort();
 558 }
 559
 560 void urcu_bp_register_thread(void)
 561 {
 562         if (caa_unlikely(!URCU_TLS(urcu_bp_reader)))
 563                 urcu_bp_register(); /* If not yet registered. */
 564 }
 565
 566 /* Disable signals, take mutex, remove from registry */
 567 static
 568 void urcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
 569 {
 570         sigset_t newmask, oldmask;
 571         int ret;
 572
 573         ret = sigfillset(&newmask);
 574         if (ret)
 575                 abort();
 576         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 577         if (ret)
 578                 abort();
 579
 580         mutex_lock(&rcu_registry_lock);
 581         remove_thread(rcu_reader_reg);
 582         mutex_unlock(&rcu_registry_lock);
 583         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 584         if (ret)
 585                 abort();
 586         urcu_bp_exit();
 587 }
 588
 589 /*
 590  * Remove thread from the registry when it exits, and flag it as
 591  * destroyed so garbage collection can take care of it.
 592  */
 593 static
 594 void urcu_bp_thread_exit_notifier(void *rcu_key)
 595 {
 596         urcu_bp_unregister(rcu_key);
 597 }
 598
 599 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 600 static
 601 void urcu_bp_sys_membarrier_status(bool available)
 602 {
 603         if (!available)
 604                 abort();
 605 }
 606 #else
 607 static
 608 void urcu_bp_sys_membarrier_status(bool available)
 609 {
 610         if (!available)
 611                 return;
 612         urcu_bp_has_sys_membarrier = 1;
 613 }
 614 #endif
 615
 616 static
 617 void urcu_bp_sys_membarrier_init(void)
 618 {
 619         bool available = false;
 620         int mask;
 621
 622         mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
 623         if (mask >= 0) {
 624                 if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
 625                         if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
 626                                 urcu_die(errno);
 627                         available = true;
 628                 }
 629         }
 630         urcu_bp_sys_membarrier_status(available);
 631 }
 632
 633 static
 634 void _urcu_bp_init(void)
 635 {
 636         mutex_lock(&init_lock);
 637         if (!urcu_bp_refcount++) {
 638                 int ret;
 639
 640                 ret = pthread_key_create(&urcu_bp_key,
 641                                 urcu_bp_thread_exit_notifier);
 642                 if (ret)
 643                         abort();
 644                 urcu_bp_sys_membarrier_init();
 645                 initialized = 1;
 646         }
 647         mutex_unlock(&init_lock);
 648 }
 649
 650 static
 651 void urcu_bp_exit(void)
 652 {
 653         mutex_lock(&init_lock);
 654         if (!--urcu_bp_refcount) {
 655                 struct registry_chunk *chunk, *tmp;
 656                 int ret;
 657
 658                 cds_list_for_each_entry_safe(chunk, tmp,
 659                                 &registry_arena.chunk_list, node) {
 660                         munmap((void *) chunk, chunk_allocation_size(chunk->capacity));
 661                 }
 662                 CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);
 663                 ret = pthread_key_delete(urcu_bp_key);
 664                 if (ret)
 665                         abort();
 666         }
 667         mutex_unlock(&init_lock);
 668 }
 669
 670 static
 671 void urcu_bp_exit_destructor(void)
 672 {
 673         urcu_call_rcu_exit();
 674         urcu_bp_exit();
 675 }
 676
 677 /*
 678  * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
 679  * sure we fork() don't race with a concurrent thread executing with
 680  * any of those locks held. This ensures that the registry and data
 681  * protected by rcu_gp_lock are in a coherent state in the child.
 682  */
 683 void urcu_bp_before_fork(void)
 684 {
 685         sigset_t newmask, oldmask;
 686         int ret;
 687
 688         ret = sigfillset(&newmask);
 689         urcu_posix_assert(!ret);
 690         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 691         urcu_posix_assert(!ret);
 692         mutex_lock(&rcu_gp_lock);
 693         mutex_lock(&rcu_registry_lock);
 694         saved_fork_signal_mask = oldmask;
 695 }
 696
 697 void urcu_bp_after_fork_parent(void)
 698 {
 699         sigset_t oldmask;
 700         int ret;
 701
 702         oldmask = saved_fork_signal_mask;
 703         mutex_unlock(&rcu_registry_lock);
 704         mutex_unlock(&rcu_gp_lock);
 705         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 706         urcu_posix_assert(!ret);
 707 }
 708
 709 /*
 710  * Prune all entries from registry except our own thread. Fits the Linux
 711  * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
 712  */
 713 static
 714 void urcu_bp_prune_registry(void)
 715 {
 716         struct registry_chunk *chunk;
 717
 718         cds_list_for_each_entry(chunk, &registry_arena.chunk_list, node) {
 719                 size_t spot_idx;
 720
 721                 for (spot_idx = 0; spot_idx < chunk->capacity; spot_idx++) {
 722                         struct urcu_bp_reader *reader = &chunk->readers[spot_idx];
 723
 724                         if (!reader->alloc)
 725                                 continue;
 726                         if (reader->tid == pthread_self())
 727                                 continue;
 728                         cleanup_thread(chunk, reader);
 729                 }
 730         }
 731 }
 732
 733 void urcu_bp_after_fork_child(void)
 734 {
 735         sigset_t oldmask;
 736         int ret;
 737
 738         urcu_bp_prune_registry();
 739         oldmask = saved_fork_signal_mask;
 740         mutex_unlock(&rcu_registry_lock);
 741         mutex_unlock(&rcu_gp_lock);
 742         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 743         urcu_posix_assert(!ret);
 744 }
 745
 746 void *urcu_bp_dereference_sym(void *p)
 747 {
 748         return _rcu_dereference(p);
 749 }
 750
 751 void *urcu_bp_set_pointer_sym(void **p, void *v)
 752 {
 753         cmm_wmb();
 754         uatomic_set(p, v);
 755         return v;
 756 }
 757
 758 void *urcu_bp_xchg_pointer_sym(void **p, void *v)
 759 {
 760         cmm_wmb();
 761         return uatomic_xchg(p, v);
 762 }
 763
 764 void *urcu_bp_cmpxchg_pointer_sym(void **p, void *old, void *_new)
 765 {
 766         cmm_wmb();
 767         return uatomic_cmpxchg(p, old, _new);
 768 }
 769
 770 DEFINE_RCU_FLAVOR(rcu_flavor);
 771
 772 #include "urcu-call-rcu-impl.h"
 773 #include "urcu-defer-impl.h"
 774 #include "urcu-poll-impl.h"