urcu/annotate: Add CMM annotation
[userspace-rcu.git] / include / urcu / static / urcu-qsbr.h
1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
3 //
4 // SPDX-License-Identifier: LGPL-2.1-or-later
5
6 #ifndef _URCU_QSBR_STATIC_H
7 #define _URCU_QSBR_STATIC_H
8
9 /*
10 * Userspace RCU QSBR header.
11 *
12 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
13 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
14 *
15 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
16 */
17
18 #include <stdlib.h>
19 #include <pthread.h>
20 #include <limits.h>
21 #include <unistd.h>
22 #include <stdint.h>
23
24 #include <urcu/annotate.h>
25 #include <urcu/debug.h>
26 #include <urcu/compiler.h>
27 #include <urcu/arch.h>
28 #include <urcu/system.h>
29 #include <urcu/uatomic.h>
30 #include <urcu/list.h>
31 #include <urcu/futex.h>
32 #include <urcu/tls-compat.h>
33 #include <urcu/static/urcu-common.h>
34
35 #ifdef __cplusplus
36 extern "C" {
37 #endif
38
39 /*
40 * This code section can only be included in LGPL 2.1 compatible source code.
41 * See below for the function call wrappers which can be used in code meant to
42 * be only linked with the Userspace RCU library. This comes with a small
43 * performance degradation on the read-side due to the added function calls.
44 * This is required to permit relinking with newer versions of the library.
45 */
46
47 #define URCU_QSBR_GP_ONLINE (1UL << 0)
48 #define URCU_QSBR_GP_CTR (1UL << 1)
49
50 extern struct urcu_gp urcu_qsbr_gp;
51
52 struct urcu_qsbr_reader {
53 /* Data used by both reader and synchronize_rcu() */
54 unsigned long ctr;
55 /* Data used for registry */
56 struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
57 int waiting;
58 pthread_t tid;
59 /* Reader registered flag, for internal checks. */
60 unsigned int registered:1;
61 };
62
63 extern DECLARE_URCU_TLS(struct urcu_qsbr_reader, urcu_qsbr_reader);
64
65 /*
66 * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
67 */
68 static inline void urcu_qsbr_wake_up_gp(void)
69 {
70 if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(urcu_qsbr_reader).waiting))) {
71 _CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader).waiting, 0);
72 cmm_smp_mb();
73 if (uatomic_read(&urcu_qsbr_gp.futex) != -1)
74 return;
75 uatomic_set(&urcu_qsbr_gp.futex, 0);
76 /*
77 * Ignoring return value until we can make this function
78 * return something (because urcu_die() is not publicly
79 * exposed).
80 */
81 (void) futex_noasync(&urcu_qsbr_gp.futex, FUTEX_WAKE, 1,
82 NULL, NULL, 0);
83 }
84 }
85
86 static inline enum urcu_state urcu_qsbr_reader_state(unsigned long *ctr,
87 cmm_annotate_t *group)
88 {
89 unsigned long v;
90
91 v = uatomic_load(ctr, CMM_RELAXED);
92 cmm_annotate_group_mem_acquire(group, ctr);
93
94 if (!v)
95 return URCU_READER_INACTIVE;
96 if (v == urcu_qsbr_gp.ctr)
97 return URCU_READER_ACTIVE_CURRENT;
98 return URCU_READER_ACTIVE_OLD;
99 }
100
101 /*
102 * Enter an RCU read-side critical section.
103 *
104 * This function is less than 10 lines long. The intent is that this
105 * function meets the 10-line criterion for LGPL, allowing this function
106 * to be invoked directly from non-LGPL code.
107 */
108 static inline void _urcu_qsbr_read_lock(void)
109 {
110 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).ctr);
111 }
112
113 /*
114 * Exit an RCU read-side critical section.
115 *
116 * This function is less than 10 lines long. The intent is that this
117 * function meets the 10-line criterion for LGPL, allowing this function
118 * to be invoked directly from non-LGPL code.
119 */
120 static inline void _urcu_qsbr_read_unlock(void)
121 {
122 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).ctr);
123 }
124
125 /*
126 * Returns whether within a RCU read-side critical section.
127 *
128 * This function is less than 10 lines long. The intent is that this
129 * function meets the 10-line criterion for LGPL, allowing this function
130 * to be invoked directly from non-LGPL code.
131 */
132 static inline int _urcu_qsbr_read_ongoing(void)
133 {
134 return URCU_TLS(urcu_qsbr_reader).ctr;
135 }
136
137 /*
138 * This is a helper function for _rcu_quiescent_state().
139 * The first cmm_smp_mb() ensures memory accesses in the prior read-side
140 * critical sections are not reordered with store to
141 * URCU_TLS(urcu_qsbr_reader).ctr, and ensures that mutexes held within an
142 * offline section that would happen to end with this
143 * urcu_qsbr_quiescent_state() call are not reordered with
144 * store to URCU_TLS(urcu_qsbr_reader).ctr.
145 */
146 static inline void _urcu_qsbr_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
147 {
148 uatomic_store(&URCU_TLS(urcu_qsbr_reader).ctr, gp_ctr, CMM_SEQ_CST);
149
150 /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
151 urcu_qsbr_wake_up_gp();
152 cmm_smp_mb();
153 }
154
155 /*
156 * Inform RCU of a quiescent state.
157 *
158 * This function is less than 10 lines long. The intent is that this
159 * function meets the 10-line criterion for LGPL, allowing this function
160 * to be invoked directly from non-LGPL code.
161 *
162 * We skip the memory barriers and gp store if our local ctr already
163 * matches the global urcu_qsbr_gp.ctr value: this is OK because a prior
164 * _rcu_quiescent_state() or _rcu_thread_online() already updated it
165 * within our thread, so we have no quiescent state to report.
166 */
167 static inline void _urcu_qsbr_quiescent_state(void)
168 {
169 unsigned long gp_ctr;
170
171 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
172 gp_ctr = uatomic_load(&urcu_qsbr_gp.ctr, CMM_RELAXED);
173 if (gp_ctr == URCU_TLS(urcu_qsbr_reader).ctr)
174 return;
175 _urcu_qsbr_quiescent_state_update_and_wakeup(gp_ctr);
176 }
177
178 /*
179 * Take a thread offline, prohibiting it from entering further RCU
180 * read-side critical sections.
181 *
182 * This function is less than 10 lines long. The intent is that this
183 * function meets the 10-line criterion for LGPL, allowing this function
184 * to be invoked directly from non-LGPL code.
185 */
186 static inline void _urcu_qsbr_thread_offline(void)
187 {
188 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
189 uatomic_store(&URCU_TLS(urcu_qsbr_reader).ctr, 0, CMM_SEQ_CST);
190 /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
191 urcu_qsbr_wake_up_gp();
192 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
193 }
194
195 /*
196 * Bring a thread online, allowing it to once again enter RCU
197 * read-side critical sections.
198 *
199 * This function is less than 10 lines long. The intent is that this
200 * function meets the 10-line criterion for LGPL, allowing this function
201 * to be invoked directly from non-LGPL code.
202 */
203 static inline void _urcu_qsbr_thread_online(void)
204 {
205 unsigned long *pctr = &URCU_TLS(urcu_qsbr_reader).ctr;
206 unsigned long ctr;
207
208 urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
209 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
210 ctr = uatomic_load(&urcu_qsbr_gp.ctr, CMM_RELAXED);
211 cmm_annotate_mem_acquire(&urcu_qsbr_gp.ctr);
212 uatomic_store(pctr, ctr, CMM_RELAXED);
213 cmm_smp_mb();
214 }
215
216 #ifdef __cplusplus
217 }
218 #endif
219
220 #endif /* _URCU_QSBR_STATIC_H */
This page took 0.034138 seconds and 4 git commands to generate.