1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
4 // SPDX-License-Identifier: LGPL-2.1-or-later
6 #ifndef _URCU_QSBR_STATIC_H
7 #define _URCU_QSBR_STATIC_H
10 * Userspace RCU QSBR header.
12 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
13 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
15 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
24 #include <urcu/annotate.h>
25 #include <urcu/debug.h>
26 #include <urcu/compiler.h>
27 #include <urcu/arch.h>
28 #include <urcu/system.h>
29 #include <urcu/uatomic.h>
30 #include <urcu/list.h>
31 #include <urcu/futex.h>
32 #include <urcu/tls-compat.h>
33 #include <urcu/static/urcu-common.h>
40 * This code section can only be included in LGPL 2.1 compatible source code.
41 * See below for the function call wrappers which can be used in code meant to
42 * be only linked with the Userspace RCU library. This comes with a small
43 * performance degradation on the read-side due to the added function calls.
44 * This is required to permit relinking with newer versions of the library.
47 #define URCU_QSBR_GP_ONLINE (1UL << 0)
48 #define URCU_QSBR_GP_CTR (1UL << 1)
50 extern struct urcu_gp urcu_qsbr_gp
;
52 struct urcu_qsbr_reader
{
53 /* Data used by both reader and synchronize_rcu() */
55 /* Data used for registry */
56 struct cds_list_head node
__attribute__((aligned(CAA_CACHE_LINE_SIZE
)));
59 /* Reader registered flag, for internal checks. */
60 unsigned int registered
:1;
63 extern DECLARE_URCU_TLS(struct urcu_qsbr_reader
, urcu_qsbr_reader
);
66 * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
68 static inline void urcu_qsbr_wake_up_gp(void)
70 if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(urcu_qsbr_reader
).waiting
))) {
71 _CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader
).waiting
, 0);
73 if (uatomic_read(&urcu_qsbr_gp
.futex
) != -1)
75 uatomic_set(&urcu_qsbr_gp
.futex
, 0);
77 * Ignoring return value until we can make this function
78 * return something (because urcu_die() is not publicly
81 (void) futex_noasync(&urcu_qsbr_gp
.futex
, FUTEX_WAKE
, 1,
86 static inline enum urcu_state
urcu_qsbr_reader_state(unsigned long *ctr
,
87 cmm_annotate_t
*group
)
91 v
= uatomic_load(ctr
, CMM_RELAXED
);
92 cmm_annotate_group_mem_acquire(group
, ctr
);
95 return URCU_READER_INACTIVE
;
96 if (v
== urcu_qsbr_gp
.ctr
)
97 return URCU_READER_ACTIVE_CURRENT
;
98 return URCU_READER_ACTIVE_OLD
;
102 * Enter an RCU read-side critical section.
104 * This function is less than 10 lines long. The intent is that this
105 * function meets the 10-line criterion for LGPL, allowing this function
106 * to be invoked directly from non-LGPL code.
108 static inline void _urcu_qsbr_read_lock(void)
110 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader
).ctr
);
114 * Exit an RCU read-side critical section.
116 * This function is less than 10 lines long. The intent is that this
117 * function meets the 10-line criterion for LGPL, allowing this function
118 * to be invoked directly from non-LGPL code.
120 static inline void _urcu_qsbr_read_unlock(void)
122 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader
).ctr
);
126 * Returns whether within a RCU read-side critical section.
128 * This function is less than 10 lines long. The intent is that this
129 * function meets the 10-line criterion for LGPL, allowing this function
130 * to be invoked directly from non-LGPL code.
132 static inline int _urcu_qsbr_read_ongoing(void)
134 return URCU_TLS(urcu_qsbr_reader
).ctr
;
138 * This is a helper function for _rcu_quiescent_state().
139 * The first cmm_smp_mb() ensures memory accesses in the prior read-side
140 * critical sections are not reordered with store to
141 * URCU_TLS(urcu_qsbr_reader).ctr, and ensures that mutexes held within an
142 * offline section that would happen to end with this
143 * urcu_qsbr_quiescent_state() call are not reordered with
144 * store to URCU_TLS(urcu_qsbr_reader).ctr.
146 static inline void _urcu_qsbr_quiescent_state_update_and_wakeup(unsigned long gp_ctr
)
148 uatomic_store(&URCU_TLS(urcu_qsbr_reader
).ctr
, gp_ctr
, CMM_SEQ_CST
);
150 /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
151 urcu_qsbr_wake_up_gp();
156 * Inform RCU of a quiescent state.
158 * This function is less than 10 lines long. The intent is that this
159 * function meets the 10-line criterion for LGPL, allowing this function
160 * to be invoked directly from non-LGPL code.
162 * We skip the memory barriers and gp store if our local ctr already
163 * matches the global urcu_qsbr_gp.ctr value: this is OK because a prior
164 * _rcu_quiescent_state() or _rcu_thread_online() already updated it
165 * within our thread, so we have no quiescent state to report.
167 static inline void _urcu_qsbr_quiescent_state(void)
169 unsigned long gp_ctr
;
171 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader
).registered
);
172 gp_ctr
= uatomic_load(&urcu_qsbr_gp
.ctr
, CMM_RELAXED
);
173 if (gp_ctr
== URCU_TLS(urcu_qsbr_reader
).ctr
)
175 _urcu_qsbr_quiescent_state_update_and_wakeup(gp_ctr
);
179 * Take a thread offline, prohibiting it from entering further RCU
180 * read-side critical sections.
182 * This function is less than 10 lines long. The intent is that this
183 * function meets the 10-line criterion for LGPL, allowing this function
184 * to be invoked directly from non-LGPL code.
186 static inline void _urcu_qsbr_thread_offline(void)
188 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader
).registered
);
189 uatomic_store(&URCU_TLS(urcu_qsbr_reader
).ctr
, 0, CMM_SEQ_CST
);
190 /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
191 urcu_qsbr_wake_up_gp();
192 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
196 * Bring a thread online, allowing it to once again enter RCU
197 * read-side critical sections.
199 * This function is less than 10 lines long. The intent is that this
200 * function meets the 10-line criterion for LGPL, allowing this function
201 * to be invoked directly from non-LGPL code.
203 static inline void _urcu_qsbr_thread_online(void)
205 unsigned long *pctr
= &URCU_TLS(urcu_qsbr_reader
).ctr
;
208 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader
).registered
);
209 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
210 ctr
= uatomic_load(&urcu_qsbr_gp
.ctr
, CMM_RELAXED
);
211 cmm_annotate_mem_acquire(&urcu_qsbr_gp
.ctr
);
212 uatomic_store(pctr
, ctr
, CMM_RELAXED
);
220 #endif /* _URCU_QSBR_STATIC_H */