From 3745305bf09e7825e75ee5b5490347ee67c6efdd Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 22 Dec 2017 10:57:59 -0500 Subject: [PATCH 1/1] liburcu-bp: Use membarrier private expedited when available For the liburcu-bp flavor, use the membarrier private expedited command when available. It is faster than the shared expedited command, but has only been introduced in 4.14 Linux kernels. When configured with --disable-sys-membarrier-fallback, liburcu-bp will abort if running on a kernel that do not provide the membarrier private expedited command (e.g. CONFIG_MEMBARRIER=n or kernel version below 4.14). Signed-off-by: Mathieu Desnoyers --- src/urcu-bp.c | 52 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/urcu-bp.c b/src/urcu-bp.c index ebe96c5..5d07f04 100644 --- a/src/urcu-bp.c +++ b/src/urcu-bp.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "urcu/arch.h" @@ -102,8 +103,12 @@ int rcu_bp_refcount; #endif enum membarrier_cmd { - MEMBARRIER_CMD_QUERY = 0, - MEMBARRIER_CMD_SHARED = (1 << 0), + MEMBARRIER_CMD_QUERY = 0, + MEMBARRIER_CMD_SHARED = (1 << 0), + /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */ + /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */ + MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3), + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4), }; static @@ -192,10 +197,12 @@ static void mutex_unlock(pthread_mutex_t *mutex) static void smp_mb_master(void) { - if (caa_likely(urcu_bp_has_sys_membarrier)) - (void) membarrier(MEMBARRIER_CMD_SHARED, 0); - else + if (caa_likely(urcu_bp_has_sys_membarrier)) { + if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0)) + urcu_die(errno); + } else { cmm_smp_mb(); + } } /* @@ -584,25 +591,38 @@ void urcu_bp_thread_exit_notifier(void *rcu_key) #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER static -void rcu_sys_membarrier_status(int available) +void rcu_sys_membarrier_status(bool available) { if (!available) abort(); } #else static -void rcu_sys_membarrier_status(int available) +void rcu_sys_membarrier_status(bool available) { - /* - * membarrier has blocking behavior, which changes the - * application behavior too much compared to using barriers when - * synchronize_rcu is used repeatedly (without using call_rcu). - * Don't use membarrier for now, unless its use has been - * explicitly forced when building liburcu. - */ + if (!available) + return; + urcu_bp_has_sys_membarrier = 1; } #endif +static +void rcu_sys_membarrier_init(void) +{ + bool available = false; + int mask; + + mask = membarrier(MEMBARRIER_CMD_QUERY, 0); + if (mask >= 0) { + if (mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) { + if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0)) + urcu_die(errno); + available = true; + } + } + rcu_sys_membarrier_status(available); +} + static void rcu_bp_init(void) { @@ -614,9 +634,7 @@ void rcu_bp_init(void) urcu_bp_thread_exit_notifier); if (ret) abort(); - ret = membarrier(MEMBARRIER_CMD_QUERY, 0); - rcu_sys_membarrier_status(ret >= 0 - && (ret & MEMBARRIER_CMD_SHARED)); + rcu_sys_membarrier_init(); initialized = 1; } mutex_unlock(&init_lock); -- 2.34.1