summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
82faadb)
Add
wmc(), rmc() and mc() which turns into barrier() or cache flush depending on the
architecture.
mb(), rmb() and wmb() turns into memory barriers or cache flush depending on the
architecture.
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
* Wait for sighandler (and thus mb()) to execute on every thread.
* BUSY-LOOP.
*/
* Wait for sighandler (and thus mb()) to execute on every thread.
* BUSY-LOOP.
*/
- while (sig_done < 1)
- smp_rmb(); /* ensure we re-read sig-done */
+ while (LOAD_REMOTE(sig_done) < 1)
+ cpu_relax();
smp_mb(); /* read sig_done before ending the barrier */
}
smp_mb(); /* read sig_done before ending the barrier */
}
if (!reader_data)
return;
sig_done = 0;
if (!reader_data)
return;
sig_done = 0;
- smp_mb(); /* write sig_done before sending the signals */
+ /*
+ * pthread_kill has a smp_mb(). But beware, we assume it performs
+ * a cache flush on architectures with non-coherent cache.
+ * smp_mb(); write sig_done before sending the signals
+ */
for (index = reader_data; index < reader_data + num_readers; index++)
pthread_kill(index->tid, SIGURCU);
/*
* Wait for sighandler (and thus mb()) to execute on every thread.
* BUSY-LOOP.
*/
for (index = reader_data; index < reader_data + num_readers; index++)
pthread_kill(index->tid, SIGURCU);
/*
* Wait for sighandler (and thus mb()) to execute on every thread.
* BUSY-LOOP.
*/
- while (sig_done < num_readers)
- smp_rmb(); /* ensure we re-read sig-done */
+ while (LOAD_REMOTE(sig_done) < num_readers)
+ cpu_relax();
smp_mb(); /* read sig_done before ending the barrier */
}
#endif
smp_mb(); /* read sig_done before ending the barrier */
}
#endif
* waiting forever while new readers are always accessing data (no
* progress).
*/
* waiting forever while new readers are always accessing data (no
* progress).
*/
/*
* Wait for previous parity to be empty of readers.
/*
* Wait for previous parity to be empty of readers.
* the writer waiting forever while new readers are always accessing
* data (no progress).
*/
* the writer waiting forever while new readers are always accessing
* data (no progress).
*/
switch_next_urcu_qparity(); /* 1 -> 0 */
switch_next_urcu_qparity(); /* 1 -> 0 */
* waiting forever while new readers are always accessing data (no
* progress).
*/
* waiting forever while new readers are always accessing data (no
* progress).
*/
/*
* Wait for previous parity to be empty of readers.
/*
* Wait for previous parity to be empty of readers.
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
+/*
+ * Assume the architecture has coherent caches. Blackfin will want this unset.
+ */
+#define CONFIG_HAVE_MEM_COHERENCY 1
+
-#define CONFIG_HAS_FENCE 1
+#define CONFIG_HAVE_FENCE 1
+
+/* Assume SMP machine, given we don't have this information */
+#define CONFIG_SMP 1
+
+
+#ifdef CONFIG_HAVE_MEM_COHERENCY
+/*
+ * Caches are coherent, no need to flush them.
+ */
+#define mc() barrier()
+#define rmc() barrier()
+#define wmc() barrier()
+#else
+#error "The architecture must create its own cache flush primitives"
+#define mc() arch_cache_flush()
+#define rmc() arch_cache_flush_read()
+#define wmc() arch_cache_flush_write()
+#endif
+
+
+#ifdef CONFIG_HAVE_MEM_COHERENCY
+#ifdef CONFIG_HAVE_FENCE
#define mb() asm volatile("mfence":::"memory")
#define rmb() asm volatile("lfence":::"memory")
#define wmb() asm volatile("sfence"::: "memory")
#define mb() asm volatile("mfence":::"memory")
#define rmb() asm volatile("lfence":::"memory")
#define wmb() asm volatile("sfence"::: "memory")
#define wmb() asm volatile("lock; addl $0,0(%%esp)"::: "memory")
#endif
#define wmb() asm volatile("lock; addl $0,0(%%esp)"::: "memory")
#endif
-/* Assume SMP machine, given we don't have this information */
-#define CONFIG_SMP 1
+#else /* !CONFIG_HAVE_MEM_COHERENCY */
+
+/*
+ * Without cache coherency, the memory barriers become cache flushes.
+ */
+#define mb() mc()
+#define rmb() rmc()
+#define wmb() wmc()
+
+#endif /* !CONFIG_HAVE_MEM_COHERENCY */
+
#ifdef CONFIG_SMP
#define smp_mb() mb()
#define smp_rmb() rmb()
#define smp_wmb() wmb()
#ifdef CONFIG_SMP
#define smp_mb() mb()
#define smp_rmb() rmb()
#define smp_wmb() wmb()
+#define smp_mc() mc()
+#define smp_rmc() rmc()
+#define smp_wmc() wmc()
#else
#define smp_mb() barrier()
#define smp_rmb() barrier()
#define smp_wmb() barrier()
#else
#define smp_mb() barrier()
#define smp_rmb() barrier()
#define smp_wmb() barrier()
+#define smp_mc() barrier()
+#define smp_rmc() barrier()
+#define smp_wmc() barrier()
+/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
+static inline void rep_nop(void)
+{
+ asm volatile("rep; nop" ::: "memory");
+}
+
+static inline void cpu_relax(void)
+{
+ rep_nop();
+}
+
static inline void atomic_inc(int *v)
{
asm volatile("lock; incl %0"
static inline void atomic_inc(int *v)
{
asm volatile("lock; incl %0"
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway
* Note 2: xchg has side effect, so that attribute volatile is necessary,
* but generally the primitive is invalid, *ptr is output argument. --ANK
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway
* Note 2: xchg has side effect, so that attribute volatile is necessary,
* but generally the primitive is invalid, *ptr is output argument. --ANK
+ * x is considered local, ptr is considered remote.
*/
static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
int size)
*/
static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
int size)
*/
#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
*/
#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
+/*
+ * Load a data from remote memory, doing a cache flush if required.
+ */
+#define LOAD_REMOTE(p) ({ \
+ smp_rmc(); \
+ typeof(p) _________p1 = ACCESS_ONCE(p); \
+ (_________p1); \
+ })
+
+/*
+ * Store v into x, where x is located in remote memory. Performs the required
+ * cache flush after writing.
+ */
+#define STORE_REMOTE(x, v) \
+ do { \
+ (x) = (v); \
+ smp_wmc; \
+ } while (0)
+
/**
* rcu_dereference - fetch an RCU-protected pointer in an
* RCU read-side critical section. This pointer may later
/**
* rcu_dereference - fetch an RCU-protected pointer in an
* RCU read-side critical section. This pointer may later
*/
#define rcu_dereference(p) ({ \
*/
#define rcu_dereference(p) ({ \
- typeof(p) _________p1 = ACCESS_ONCE(p); \
+ typeof(p) _________p1 = LOAD_REMOTE(p); \
smp_read_barrier_depends(); \
(_________p1); \
})
smp_read_barrier_depends(); \
(_________p1); \
})
#define SIGURCU SIGUSR1
/*
#define SIGURCU SIGUSR1
/*
#endif
#ifdef DEBUG_FULL_MB
#endif
#ifdef DEBUG_FULL_MB
-static inline void read_barrier()
+static inline void reader_barrier()
-static inline void read_barrier()
+static inline void reader_barrier()
* Make sure both tests below are done on the same version of *value
* to insure consistency.
*/
* Make sure both tests below are done on the same version of *value
* to insure consistency.
*/
- v = ACCESS_ONCE(*value);
+ v = LOAD_REMOTE(*value);
return (v & RCU_GP_CTR_NEST_MASK) &&
((v ^ urcu_gp_ctr) & RCU_GP_CTR_BIT);
}
return (v & RCU_GP_CTR_NEST_MASK) &&
((v ^ urcu_gp_ctr) & RCU_GP_CTR_BIT);
}
tmp = urcu_active_readers;
/* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
tmp = urcu_active_readers;
/* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
- /* The data dependency "read urcu_gp_ctr, write urcu_active_readers",
- * serializes those two memory operations. */
+ /*
+ * The data dependency "read urcu_gp_ctr, write urcu_active_readers",
+ * serializes those two memory operations. We are not using STORE_REMOTE
+ * and LOAD_REMOTE here (although we should) because the writer will
+ * wake us up with a signal which does a flush in its handler to perform
+ * urcu_gp_ctr re-read and urcu_active_readers commit to main memory.
+ */
if (likely(!(tmp & RCU_GP_CTR_NEST_MASK)))
urcu_active_readers = ACCESS_ONCE(urcu_gp_ctr);
else
if (likely(!(tmp & RCU_GP_CTR_NEST_MASK)))
urcu_active_readers = ACCESS_ONCE(urcu_gp_ctr);
else
* Increment active readers count before accessing the pointer.
* See force_mb_all_threads().
*/
* Increment active readers count before accessing the pointer.
* See force_mb_all_threads().
*/
}
static inline void rcu_read_unlock(void)
{
}
static inline void rcu_read_unlock(void)
{
/*
* Finish using rcu before decrementing the pointer.
* See force_mb_all_threads().
/*
* Finish using rcu before decrementing the pointer.
* See force_mb_all_threads().
((v) != NULL)) \
wmb(); \
(p) = (v); \
((v) != NULL)) \
wmb(); \
(p) = (v); \
})
#define rcu_xchg_pointer(p, v) \
})
#define rcu_xchg_pointer(p, v) \