+/*
+ * Wake-up any waiting defer thread. Called from many concurrent threads.
+ */
+static void wake_up_defer(void)
+{
+ if (unlikely(uatomic_read(&defer_thread_futex) == -1)) {
+ uatomic_set(&defer_thread_futex, 0);
+ futex_noasync(&defer_thread_futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0);
+ }
+}
+
+static unsigned long rcu_defer_num_callbacks(void)
+{
+ unsigned long num_items = 0, head;
+ struct defer_queue *index;
+
+ internal_urcu_lock(&urcu_defer_mutex);
+ list_for_each_entry(index, ®istry, list) {
+ head = LOAD_SHARED(index->head);
+ num_items += head - index->tail;
+ }
+ internal_urcu_unlock(&urcu_defer_mutex);
+ return num_items;
+}
+
+/*
+ * Defer thread waiting. Single thread.
+ */
+static void wait_defer(void)
+{
+ uatomic_dec(&defer_thread_futex);
+ smp_mb(); /* Write futex before read queue */
+ if (rcu_defer_num_callbacks()) {
+ smp_mb(); /* Read queue before write futex */
+ /* Callbacks are queued, don't wait. */
+ uatomic_set(&defer_thread_futex, 0);
+ } else {
+ smp_rmb(); /* Read queue before read futex */
+ if (uatomic_read(&defer_thread_futex) == -1)
+ futex_noasync(&defer_thread_futex, FUTEX_WAIT, -1,
+ NULL, NULL, 0);
+ }
+}
+