Cleanup headers

[urcu.git] / urcu-qsbr.c

diff --git a/urcu-qsbr.c b/urcu-qsbr.c
index 5c6eaa014721267da7de5f85fff22835d41a80b2..f5103eeea78351b64cb64a4cf99b2faca835d771 100644 (file)
--- a/urcu-qsbr.c
+++ b/urcu-qsbr.c
@@ -39,6 +39,8 @@
 
 static pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+int gp_futex;
+
 /*
  * Global grace period counter.
  */
@@ -99,6 +101,27 @@ static void internal_urcu_unlock(void)
        }
 }
 
+/*
+ * synchronize_rcu() waiting. Single thread.
+ */
+static void wait_gp(struct reader_registry *index)
+{
+       uatomic_dec(&gp_futex);
+       smp_mb(); /* Write futex before read reader_gp */
+       if (!rcu_gp_ongoing(index->rcu_reader_qs_gp)) {
+               /* Read reader_gp before write futex */
+               smp_mb();
+               /* Callbacks are queued, don't wait. */
+               uatomic_set(&gp_futex, 0);
+       } else {
+               /* Read reader_gp before read futex */
+               smp_rmb();
+               if (uatomic_read(&gp_futex) == -1)
+                       futex(&gp_futex, FUTEX_WAIT, -1,
+                             NULL, NULL, 0);
+       }
+}
+
 static void wait_for_quiescent_state(void)
 {
        struct reader_registry *index;
@@ -109,13 +132,19 @@ static void wait_for_quiescent_state(void)
         * Wait for each thread rcu_reader_qs_gp count to become 0.
         */
        for (index = registry; index < registry + num_readers; index++) {
+               int wait_loops = 0;
+
+               while (rcu_gp_ongoing(index->rcu_reader_qs_gp)) {
+                       if (wait_loops++ == RCU_QS_ACTIVE_ATTEMPTS) {
+                               wait_gp(index);
+                       } else {
 #ifndef HAS_INCOHERENT_CACHES
-               while (rcu_gp_ongoing(index->rcu_reader_qs_gp))
-                       cpu_relax();
+                               cpu_relax();
 #else /* #ifndef HAS_INCOHERENT_CACHES */
-               while (rcu_gp_ongoing(index->rcu_reader_qs_gp))
-                       smp_mb();
+                               smp_mb();
 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+                       }
+               }
        }
 }
 
@@ -258,13 +287,13 @@ void *rcu_assign_pointer_sym(void **p, void *v)
 void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new)
 {
        wmb();
-       return cmpxchg(p, old, _new);
+       return uatomic_cmpxchg(p, old, _new);
 }
 
 void *rcu_xchg_pointer_sym(void **p, void *v)
 {
        wmb();
-       return xchg(p, v);
+       return uatomic_xchg(p, v);
 }
 
 void *rcu_publish_content_sym(void **p, void *v)