X-Git-Url: http://git.lttng.org/?p=urcu.git;a=blobdiff_plain;f=src%2Furcu.c;h=c47f51b53330322468cd5a6b870eab612907fb68;hp=ccd9706dda6fe6a8b6bfcba2fced7893054cf3ae;hb=c0bb9f693f926595a7cb8b4ce712cef08d9f5d49;hpb=6893800a4d1cc14dff0395ddcd660a5138db183d

diff --git a/src/urcu.c b/src/urcu.c
index ccd9706..c47f51b 100644
--- a/src/urcu.c
+++ b/src/urcu.c
@@ -34,6 +34,7 @@
 #include <stdint.h>
 #include <string.h>
 #include <errno.h>
+#include <stdbool.h>
 #include <poll.h>
 
 #include "urcu/arch.h"
@@ -71,13 +72,21 @@
 #endif
 
 enum membarrier_cmd {
-	MEMBARRIER_CMD_QUERY = 0,
-	MEMBARRIER_CMD_SHARED = (1 << 0),
+	MEMBARRIER_CMD_QUERY				= 0,
+	MEMBARRIER_CMD_SHARED				= (1 << 0),
+	/* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
+	/* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
+	MEMBARRIER_CMD_PRIVATE_EXPEDITED		= (1 << 3),
+	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED	= (1 << 4),
 };
 
 #ifdef RCU_MEMBARRIER
 static int init_done;
-int rcu_has_sys_membarrier;
+static int has_sys_membarrier_private_expedited;
+
+#ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER
+int rcu_has_sys_membarrier_memb;
+#endif
 
 void __attribute__((constructor)) rcu_init(void);
 #endif
@@ -160,10 +169,14 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 #ifdef RCU_MEMBARRIER
 static void smp_mb_master(void)
 {
-	if (caa_likely(rcu_has_sys_membarrier))
-		(void) membarrier(MEMBARRIER_CMD_SHARED, 0);
-	else
+	if (caa_likely(rcu_has_sys_membarrier_memb)) {
+		if (membarrier(has_sys_membarrier_private_expedited ?
+				MEMBARRIER_CMD_PRIVATE_EXPEDITED :
+				MEMBARRIER_CMD_SHARED, 0))
+			urcu_die(errno);
+	} else {
 		cmm_smp_mb();
+	}
 }
 #endif
 
@@ -532,17 +545,50 @@ void rcu_unregister_thread(void)
 }
 
 #ifdef RCU_MEMBARRIER
-void rcu_init(void)
+
+#ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
+static
+void rcu_sys_membarrier_status(bool available)
 {
-	int ret;
+	if (!available)
+		abort();
+}
+#else
+static
+void rcu_sys_membarrier_status(bool available)
+{
+	if (!available)
+		return;
+	rcu_has_sys_membarrier_memb = 1;
+}
+#endif
+
+static
+void rcu_sys_membarrier_init(void)
+{
+	bool available = false;
+	int mask;
+
+	mask = membarrier(MEMBARRIER_CMD_QUERY, 0);
+	if (mask >= 0) {
+		if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
+			if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0))
+				urcu_die(errno);
+			has_sys_membarrier_private_expedited = 1;
+			available = true;
+		} else if (mask & MEMBARRIER_CMD_SHARED) {
+			available = true;
+		}
+	}
+	rcu_sys_membarrier_status(available);
+}
 
+void rcu_init(void)
+{
 	if (init_done)
 		return;
 	init_done = 1;
-	ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
-	if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) {
-		rcu_has_sys_membarrier = 1;
-	}
+	rcu_sys_membarrier_init();
 }
 #endif