liburcu-bp: Use membarrier private expedited when available
[urcu.git] / src / urcu-bp.c
index 81bb91a9f07ebbcea661dd159bc726851ae2f041..5d07f040f38c2ef0b33fce285c49de8e14188528 100644 (file)
@@ -33,6 +33,7 @@
 #include <errno.h>
 #include <poll.h>
 #include <unistd.h>
+#include <stdbool.h>
 #include <sys/mman.h>
 
 #include "urcu/arch.h"
@@ -102,8 +103,12 @@ int rcu_bp_refcount;
 #endif
 
 enum membarrier_cmd {
-       MEMBARRIER_CMD_QUERY = 0,
-       MEMBARRIER_CMD_SHARED = (1 << 0),
+       MEMBARRIER_CMD_QUERY                            = 0,
+       MEMBARRIER_CMD_SHARED                           = (1 << 0),
+       /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
+       /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
+       MEMBARRIER_CMD_PRIVATE_EXPEDITED                = (1 << 3),
+       MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED       = (1 << 4),
 };
 
 static
@@ -192,10 +197,12 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 
 static void smp_mb_master(void)
 {
-       if (caa_likely(urcu_bp_has_sys_membarrier))
-               (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
-       else
+       if (caa_likely(urcu_bp_has_sys_membarrier)) {
+               if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0))
+                       urcu_die(errno);
+       } else {
                cmm_smp_mb();
+       }
 }
 
 /*
@@ -374,7 +381,8 @@ void expand_arena(struct registry_arena *arena)
                        sizeof(struct registry_chunk)
                        + sizeof(struct rcu_reader));
                new_chunk_len = ARENA_INIT_ALLOC;
-               new_chunk = mmap(NULL, new_chunk_len,
+               new_chunk = (struct registry_chunk *) mmap(NULL,
+                       new_chunk_len,
                        PROT_READ | PROT_WRITE,
                        MAP_ANONYMOUS | MAP_PRIVATE,
                        -1, 0);
@@ -408,7 +416,8 @@ void expand_arena(struct registry_arena *arena)
        }
 
        /* Remap did not succeed, we need to add a new chunk. */
-       new_chunk = mmap(NULL, new_chunk_len,
+       new_chunk = (struct registry_chunk *) mmap(NULL,
+               new_chunk_len,
                PROT_READ | PROT_WRITE,
                MAP_ANONYMOUS | MAP_PRIVATE,
                -1, 0);
@@ -582,20 +591,38 @@ void urcu_bp_thread_exit_notifier(void *rcu_key)
 
 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 static
-void rcu_sys_membarrier_status(int available)
+void rcu_sys_membarrier_status(bool available)
 {
        if (!available)
                abort();
 }
 #else
 static
-void rcu_sys_membarrier_status(int available)
+void rcu_sys_membarrier_status(bool available)
 {
-       if (available)
-               urcu_bp_has_sys_membarrier = 1;
+       if (!available)
+               return;
+       urcu_bp_has_sys_membarrier = 1;
 }
 #endif
 
+static
+void rcu_sys_membarrier_init(void)
+{
+       bool available = false;
+       int mask;
+
+       mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
+       if (mask >= 0) {
+               if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
+                       if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
+                               urcu_die(errno);
+                       available = true;
+               }
+       }
+       rcu_sys_membarrier_status(available);
+}
+
 static
 void rcu_bp_init(void)
 {
@@ -607,9 +634,7 @@ void rcu_bp_init(void)
                                urcu_bp_thread_exit_notifier);
                if (ret)
                        abort();
-               ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
-               rcu_sys_membarrier_status(ret >= 0
-                               && (ret & MEMBARRIER_CMD_SHARED));
+               rcu_sys_membarrier_init();
                initialized = 1;
        }
        mutex_unlock(&init_lock);
@@ -625,7 +650,7 @@ void rcu_bp_exit(void)
 
                cds_list_for_each_entry_safe(chunk, tmp,
                                &registry_arena.chunk_list, node) {
-                       munmap(chunk, chunk->data_len
+                       munmap((void *) chunk, chunk->data_len
                                        + sizeof(struct registry_chunk));
                }
                CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);
This page took 0.032426 seconds and 4 git commands to generate.