Fix rcu_assign_pointer() dynamic linking behavior

[urcu.git] / urcu.c

diff --git a/urcu.c b/urcu.c
index d441355e8a85b85a814c778f52acba7edb47ea2b..c13ad634e7cb7b98f24d8c9a5a756eb26487bf7b 100644 (file)
--- a/urcu.c
+++ b/urcu.c
@@ -65,9 +65,6 @@ long urcu_gp_ctr = RCU_GP_COUNT;
  */
 struct urcu_reader __thread urcu_reader;
 
-/* Thread IDs of registered readers */
-#define INIT_NUM_THREADS 4
-
 #ifdef DEBUG_YIELD
 unsigned int yield_active;
 unsigned int __thread rand_yield;
@@ -122,16 +119,19 @@ static void switch_next_urcu_qparity(void)
 }
 
 #ifdef URCU_MB
+#if 0 /* unused */
 static void force_mb_single_thread(struct urcu_reader *index)
 {
        smp_mb();
 }
+#endif //0
 
 static void force_mb_all_threads(void)
 {
        smp_mb();
 }
 #else /* #ifdef URCU_MB */
+#if 0 /* unused */
 static void force_mb_single_thread(struct urcu_reader *index)
 {
        assert(!list_empty(&registry));
@@ -154,6 +154,7 @@ static void force_mb_single_thread(struct urcu_reader *index)
        }
        smp_mb();       /* read ->need_mb before ending the barrier */
 }
+#endif //0
 
 static void force_mb_all_threads(void)
 {
@@ -202,63 +203,82 @@ static void force_mb_all_threads(void)
 /*
  * synchronize_rcu() waiting. Single thread.
  */
-static void wait_gp(struct urcu_reader *index)
+static void wait_gp(void)
 {
-       uatomic_dec(&gp_futex);
-       force_mb_single_thread(index); /* Write futex before read reader_gp */
-       if (!rcu_old_gp_ongoing(&index->ctr)) {
-               /* Read reader_gp before write futex */
-               force_mb_single_thread(index);
-               /* Callbacks are queued, don't wait. */
-               uatomic_set(&gp_futex, 0);
-       } else {
-               /* Read reader_gp before read futex */
-               force_mb_single_thread(index);
-               if (uatomic_read(&gp_futex) == -1)
-                       futex(&gp_futex, FUTEX_WAIT, -1,
-                             NULL, NULL, 0);
-       }
+       /* Read reader_gp before read futex */
+       force_mb_all_threads();
+       if (uatomic_read(&gp_futex) == -1)
+               futex(&gp_futex, FUTEX_WAIT, -1,
+                     NULL, NULL, 0);
 }
 
 void wait_for_quiescent_state(void)
 {
-       struct urcu_reader *index;
+       LIST_HEAD(qsreaders);
+       int wait_loops = 0;
+       struct urcu_reader *index, *tmp;
 
        if (list_empty(&registry))
                return;
        /*
         * Wait for each thread urcu_reader.ctr count to become 0.
         */
-       list_for_each_entry(index, &registry, head) {
-               int wait_loops = 0;
+       for (;;) {
+               wait_loops++;
+               if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
+                       uatomic_dec(&gp_futex);
+                       /* Write futex before read reader_gp */
+                       force_mb_all_threads();
+               }
+
+               list_for_each_entry_safe(index, tmp, &registry, head) {
+                       if (!rcu_old_gp_ongoing(&index->ctr))
+                               list_move(&index->head, &qsreaders);
+               }
+
 #ifndef HAS_INCOHERENT_CACHES
-               while (rcu_old_gp_ongoing(&index->ctr)) {
-                       if (wait_loops++ == RCU_QS_ACTIVE_ATTEMPTS) {
-                               wait_gp(index);
-                       } else {
-                               cpu_relax();
+               if (list_empty(&registry)) {
+                       if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
+                               /* Read reader_gp before write futex */
+                               force_mb_all_threads();
+                               uatomic_set(&gp_futex, 0);
                        }
+                       break;
+               } else {
+                       if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
+                               wait_gp();
+                       else
+                               cpu_relax();
                }
 #else /* #ifndef HAS_INCOHERENT_CACHES */
                /*
                 * BUSY-LOOP. Force the reader thread to commit its
                 * urcu_reader.ctr update to memory if we wait for too long.
                 */
-               while (rcu_old_gp_ongoing(&index->ctr)) {
-                       switch (wait_loops++) {
+               if (list_empty(&registry)) {
+                       if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
+                               /* Read reader_gp before write futex */
+                               force_mb_all_threads();
+                               uatomic_set(&gp_futex, 0);
+                       }
+                       break;
+               } else {
+                       switch (wait_loops) {
                        case RCU_QS_ACTIVE_ATTEMPTS:
-                               wait_gp(index);
-                               break;
+                               wait_gp();
+                               break; /* only escape switch */
                        case KICK_READER_LOOPS:
-                               force_mb_single_thread(index);
+                               force_mb_all_threads();
                                wait_loops = 0;
-                               break;
+                               break; /* only escape switch */
                        default:
                                cpu_relax();
                        }
                }
 #endif /* #else #ifndef HAS_INCOHERENT_CACHES */
        }
+       /* put back the reader list in the registry */
+       list_splice(&qsreaders, &registry);
 }
 
 void synchronize_rcu(void)
@@ -360,7 +380,7 @@ void *rcu_dereference(void *p)
 void *rcu_assign_pointer_sym(void **p, void *v)
 {
        wmb();
-       return STORE_SHARED(p, v);
+       return STORE_SHARED(*p, v);
 }
 
 void *rcu_xchg_pointer_sym(void **p, void *v)
@@ -386,11 +406,12 @@ void *rcu_publish_content_sym(void **p, void *v)
 
 void rcu_register_thread(void)
 {
-       internal_urcu_lock();
-       urcu_init();    /* In case gcc does not support constructor attribute */
        urcu_reader.tid = pthread_self();
        assert(urcu_reader.need_mb == 0);
        assert(urcu_reader.ctr == 0);
+
+       internal_urcu_lock();
+       urcu_init();    /* In case gcc does not support constructor attribute */
        list_add(&urcu_reader.head, &registry);
        internal_urcu_unlock();
 }