+ cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
+ switch (rcu_reader_state(&index->ctr)) {
+ case RCU_READER_ACTIVE_CURRENT:
+ if (cur_snap_readers) {
+ cds_list_move(&index->node,
+ cur_snap_readers);
+ break;
+ }
+ /* Fall-through */
+ case RCU_READER_INACTIVE:
+ cds_list_move(&index->node, qsreaders);
+ break;
+ case RCU_READER_ACTIVE_OLD:
+ /*
+ * Old snapshot. Leaving node in
+ * input_readers will make us busy-loop
+ * until the snapshot becomes current or
+ * the reader becomes inactive.
+ */
+ break;
+ }
+ }
+
+#ifndef HAS_INCOHERENT_CACHES
+ if (cds_list_empty(input_readers)) {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* Read reader_gp before write futex */
+ smp_mb_master(RCU_MB_GROUP);
+ uatomic_set(&rcu_gp.futex, 0);
+ }
+ break;
+ } else {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* wait_gp unlocks/locks registry lock. */
+ wait_gp();
+ } else {
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ caa_cpu_relax();
+ /*
+ * Re-lock the registry lock before the
+ * next loop.
+ */
+ mutex_lock(&rcu_registry_lock);
+ }
+ }
+#else /* #ifndef HAS_INCOHERENT_CACHES */
+ /*
+ * BUSY-LOOP. Force the reader thread to commit its
+ * URCU_TLS(rcu_reader).ctr update to memory if we wait
+ * for too long.
+ */
+ if (cds_list_empty(input_readers)) {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* Read reader_gp before write futex */
+ smp_mb_master(RCU_MB_GROUP);
+ uatomic_set(&rcu_gp.futex, 0);
+ }
+ break;
+ } else {
+ if (wait_gp_loops == KICK_READER_LOOPS) {
+ smp_mb_master(RCU_MB_GROUP);
+ wait_gp_loops = 0;
+ }
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* wait_gp unlocks/locks registry lock. */
+ wait_gp();
+ wait_gp_loops++;
+ } else {
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ caa_cpu_relax();
+ /*
+ * Re-lock the registry lock before the
+ * next loop.
+ */
+ mutex_lock(&rcu_registry_lock);
+ }
+ }
+#endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+ }
+}
+
+void synchronize_rcu(void)
+{
+ CDS_LIST_HEAD(cur_snap_readers);
+ CDS_LIST_HEAD(qsreaders);
+ DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
+ struct urcu_waiters waiters;
+
+ /*
+ * Add ourself to gp_waiters queue of threads awaiting to wait
+ * for a grace period. Proceed to perform the grace period only
+ * if we are the first thread added into the queue.
+ * The implicit memory barrier before urcu_wait_add()
+ * orders prior memory accesses of threads put into the wait
+ * queue before their insertion into the wait queue.
+ */
+ if (urcu_wait_add(&gp_waiters, &wait) != 0) {
+ /* Not first in queue: will be awakened by another thread. */
+ urcu_adaptative_busy_wait(&wait);
+ /* Order following memory accesses after grace period. */
+ cmm_smp_mb();
+ return;