urcu/annotate: Add CMM annotation
[userspace-rcu.git] / include / urcu / static / urcu-memb.h
1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
3 //
4 // SPDX-License-Identifier: LGPL-2.1-or-later
5
6 #ifndef _URCU_MEMB_STATIC_H
7 #define _URCU_MEMB_STATIC_H
8
9 /*
10 * Userspace RCU header.
11 *
12 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
13 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
14 *
15 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
16 */
17
18 #include <stdlib.h>
19 #include <pthread.h>
20 #include <unistd.h>
21 #include <stdint.h>
22
23 #include <urcu/annotate.h>
24 #include <urcu/debug.h>
25 #include <urcu/config.h>
26 #include <urcu/compiler.h>
27 #include <urcu/arch.h>
28 #include <urcu/system.h>
29 #include <urcu/uatomic.h>
30 #include <urcu/list.h>
31 #include <urcu/futex.h>
32 #include <urcu/tls-compat.h>
33 #include <urcu/static/urcu-common.h>
34
35 #ifdef __cplusplus
36 extern "C" {
37 #endif
38
39 /*
40 * This code section can only be included in LGPL 2.1 compatible source code.
41 * See below for the function call wrappers which can be used in code meant to
42 * be only linked with the Userspace RCU library. This comes with a small
43 * performance degradation on the read-side due to the added function calls.
44 * This is required to permit relinking with newer versions of the library.
45 */
46
47 /*
48 * Slave barriers are only guaranteed to be ordered wrt master barriers.
49 *
50 * The pair ordering is detailed as (O: ordered, X: not ordered) :
51 * slave master
52 * slave X O
53 * master O O
54 */
55
56 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
57 #define urcu_memb_has_sys_membarrier 1
58 #else
59 extern int urcu_memb_has_sys_membarrier;
60 #endif
61
62 static inline void urcu_memb_smp_mb_slave(void)
63 {
64 if (caa_likely(urcu_memb_has_sys_membarrier))
65 cmm_barrier();
66 else
67 cmm_smp_mb();
68 }
69
70 extern struct urcu_gp urcu_memb_gp;
71
72 extern DECLARE_URCU_TLS(struct urcu_reader, urcu_memb_reader);
73
74 /*
75 * Helper for _rcu_read_lock(). The format of urcu_memb_gp.ctr (as well as
76 * the per-thread rcu_reader.ctr) has the lower-order bits containing a count of
77 * _rcu_read_lock() nesting, and a single high-order URCU_BP_GP_CTR_PHASE bit
78 * that contains either zero or one. The smp_mb_slave() ensures that the accesses in
79 * _rcu_read_lock() happen before the subsequent read-side critical section.
80 */
81 static inline void _urcu_memb_read_lock_update(unsigned long tmp)
82 {
83 unsigned long *ctr = &URCU_TLS(urcu_memb_reader).ctr;
84
85 if (caa_likely(!(tmp & URCU_GP_CTR_NEST_MASK))) {
86 unsigned long *pgctr = &urcu_memb_gp.ctr;
87 unsigned long gctr = uatomic_load(pgctr, CMM_RELAXED);
88
89 /* Paired with following mb slave. */
90 cmm_annotate_mem_acquire(pgctr);
91 uatomic_store(ctr, gctr, CMM_RELAXED);
92
93 urcu_memb_smp_mb_slave();
94 } else {
95 uatomic_store(ctr, tmp + URCU_GP_COUNT, CMM_RELAXED);
96 }
97 }
98
99 /*
100 * Enter an RCU read-side critical section.
101 *
102 * The first cmm_barrier() call ensures that the compiler does not reorder
103 * the body of _rcu_read_lock() with a mutex.
104 *
105 * This function and its helper are both less than 10 lines long. The
106 * intent is that this function meets the 10-line criterion in LGPL,
107 * allowing this function to be invoked directly from non-LGPL code.
108 */
109 static inline void _urcu_memb_read_lock(void)
110 {
111 unsigned long tmp;
112
113 urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
114 cmm_barrier();
115 tmp = URCU_TLS(urcu_memb_reader).ctr;
116 urcu_assert_debug((tmp & URCU_GP_CTR_NEST_MASK) != URCU_GP_CTR_NEST_MASK);
117 _urcu_memb_read_lock_update(tmp);
118 }
119
120 /*
121 * This is a helper function for _rcu_read_unlock().
122 *
123 * The first smp_mb_slave() call ensures that the critical section is
124 * seen to precede the store to rcu_reader.ctr.
125 * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr
126 * before reading the update-side futex.
127 */
128 static inline void _urcu_memb_read_unlock_update_and_wakeup(unsigned long tmp)
129 {
130 unsigned long *ctr = &URCU_TLS(urcu_memb_reader).ctr;
131
132 if (caa_likely((tmp & URCU_GP_CTR_NEST_MASK) == URCU_GP_COUNT)) {
133 urcu_memb_smp_mb_slave();
134 cmm_annotate_mem_release(ctr);
135 uatomic_store(ctr, tmp - URCU_GP_COUNT, CMM_RELAXED);
136 urcu_memb_smp_mb_slave();
137 urcu_common_wake_up_gp(&urcu_memb_gp);
138 } else {
139 uatomic_store(ctr, tmp - URCU_GP_COUNT, CMM_RELAXED);
140 }
141 }
142
143 /*
144 * Exit an RCU read-side critical section. Both this function and its
145 * helper are smaller than 10 lines of code, and are intended to be
146 * usable by non-LGPL code, as called out in LGPL.
147 */
148 static inline void _urcu_memb_read_unlock(void)
149 {
150 unsigned long tmp;
151
152 urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
153 tmp = URCU_TLS(urcu_memb_reader).ctr;
154 urcu_assert_debug(tmp & URCU_GP_CTR_NEST_MASK);
155 _urcu_memb_read_unlock_update_and_wakeup(tmp);
156 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
157 }
158
159 /*
160 * Returns whether within a RCU read-side critical section.
161 *
162 * This function is less than 10 lines long. The intent is that this
163 * function meets the 10-line criterion for LGPL, allowing this function
164 * to be invoked directly from non-LGPL code.
165 */
166 static inline int _urcu_memb_read_ongoing(void)
167 {
168 return URCU_TLS(urcu_memb_reader).ctr & URCU_GP_CTR_NEST_MASK;
169 }
170
171 #ifdef __cplusplus
172 }
173 #endif
174
175 #endif /* _URCU_MEMB_STATIC_H */
This page took 0.032163 seconds and 4 git commands to generate.