X-Git-Url: https://git.lttng.org/?p=urcu.git;a=blobdiff_plain;f=src%2Furcu-bp.c;h=5d07f040f38c2ef0b33fce285c49de8e14188528;hp=4b4fe45b2c4bcaefe3dde281d0074f87caf0bc6b;hb=3745305bf09e7825e75ee5b5490347ee67c6efdd;hpb=6893800a4d1cc14dff0395ddcd660a5138db183d

diff --git a/src/urcu-bp.c b/src/urcu-bp.c
index 4b4fe45..5d07f04 100644
--- a/src/urcu-bp.c
+++ b/src/urcu-bp.c
@@ -33,6 +33,7 @@
 #include <errno.h>
 #include <poll.h>
 #include <unistd.h>
+#include <stdbool.h>
 #include <sys/mman.h>
 
 #include "urcu/arch.h"
@@ -102,8 +103,12 @@ int rcu_bp_refcount;
 #endif
 
 enum membarrier_cmd {
-	MEMBARRIER_CMD_QUERY = 0,
-	MEMBARRIER_CMD_SHARED = (1 << 0),
+	MEMBARRIER_CMD_QUERY				= 0,
+	MEMBARRIER_CMD_SHARED				= (1 << 0),
+	/* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
+	/* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
+	MEMBARRIER_CMD_PRIVATE_EXPEDITED		= (1 << 3),
+	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED	= (1 << 4),
 };
 
 static
@@ -111,7 +116,9 @@ void __attribute__((constructor)) rcu_bp_init(void);
 static
 void __attribute__((destructor)) rcu_bp_exit(void);
 
+#ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
 int urcu_bp_has_sys_membarrier;
+#endif
 
 /*
  * rcu_gp_lock ensures mutual exclusion between threads calling
@@ -190,10 +197,12 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 
 static void smp_mb_master(void)
 {
-	if (caa_likely(urcu_bp_has_sys_membarrier))
-		(void) membarrier(MEMBARRIER_CMD_SHARED, 0);
-	else
+	if (caa_likely(urcu_bp_has_sys_membarrier)) {
+		if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0))
+			urcu_die(errno);
+	} else {
 		cmm_smp_mb();
+	}
 }
 
 /*
@@ -372,7 +381,8 @@ void expand_arena(struct registry_arena *arena)
 			sizeof(struct registry_chunk)
 			+ sizeof(struct rcu_reader));
 		new_chunk_len = ARENA_INIT_ALLOC;
-		new_chunk = mmap(NULL, new_chunk_len,
+		new_chunk = (struct registry_chunk *) mmap(NULL,
+			new_chunk_len,
 			PROT_READ | PROT_WRITE,
 			MAP_ANONYMOUS | MAP_PRIVATE,
 			-1, 0);
@@ -406,7 +416,8 @@ void expand_arena(struct registry_arena *arena)
 	}
 
 	/* Remap did not succeed, we need to add a new chunk. */
-	new_chunk = mmap(NULL, new_chunk_len,
+	new_chunk = (struct registry_chunk *) mmap(NULL,
+		new_chunk_len,
 		PROT_READ | PROT_WRITE,
 		MAP_ANONYMOUS | MAP_PRIVATE,
 		-1, 0);
@@ -578,6 +589,40 @@ void urcu_bp_thread_exit_notifier(void *rcu_key)
 	rcu_bp_unregister(rcu_key);
 }
 
+#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
+static
+void rcu_sys_membarrier_status(bool available)
+{
+	if (!available)
+		abort();
+}
+#else
+static
+void rcu_sys_membarrier_status(bool available)
+{
+	if (!available)
+		return;
+	urcu_bp_has_sys_membarrier = 1;
+}
+#endif
+
+static
+void rcu_sys_membarrier_init(void)
+{
+	bool available = false;
+	int mask;
+
+	mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
+	if (mask >= 0) {
+		if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
+			if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
+				urcu_die(errno);
+			available = true;
+		}
+	}
+	rcu_sys_membarrier_status(available);
+}
+
 static
 void rcu_bp_init(void)
 {
@@ -589,10 +634,7 @@ void rcu_bp_init(void)
 				urcu_bp_thread_exit_notifier);
 		if (ret)
 			abort();
-		ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
-		if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) {
-			urcu_bp_has_sys_membarrier = 1;
-		}
+		rcu_sys_membarrier_init();
 		initialized = 1;
 	}
 	mutex_unlock(&init_lock);
@@ -608,7 +650,7 @@ void rcu_bp_exit(void)
 
 		cds_list_for_each_entry_safe(chunk, tmp,
 				&registry_arena.chunk_list, node) {
-			munmap(chunk, chunk->data_len
+			munmap((void *) chunk, chunk->data_len
 					+ sizeof(struct registry_chunk));
 		}
 		CDS_INIT_LIST_HEAD(&registry_arena.chunk_list);