Fix: call_rcu: futex wait: handle spurious futex wakeups

[urcu.git] / src / urcu-call-rcu-impl.h

diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h
index b6ec6ba3913fd1546a39485a3764a7488e7112bb..3627e6e3adc9902bc2364f03c9c3333c1b46ef7b 100644 (file)
--- a/src/urcu-call-rcu-impl.h
+++ b/src/urcu-call-rcu-impl.h
@@ -199,7 +199,7 @@ static void call_rcu_unlock(pthread_mutex_t *pmp)
  * Losing affinity can be caused by CPU hotunplug/hotplug, or by
  * cpuset(7).
  */
-#if HAVE_SCHED_SETAFFINITY
+#ifdef HAVE_SCHED_SETAFFINITY
 static
 int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
@@ -233,7 +233,7 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 }
 #else
 static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused)))
 {
        return 0;
 }
@@ -243,17 +243,25 @@ static void call_rcu_wait(struct call_rcu_data *crdp)
 {
        /* Read call_rcu list before read futex */
        cmm_smp_mb();
-       if (uatomic_read(&crdp->futex) != -1)
-               return;
-       while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
-                       NULL, NULL, 0)) {
+       while (uatomic_read(&crdp->futex) == -1) {
+               if (!futex_async(&crdp->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+                       /*
+                        * Prior queued wakeups queued by unrelated code
+                        * using the same address can cause futex wait to
+                        * return 0 even through the futex value is still
+                        * -1 (spurious wakeups). Check the value again
+                        * in user-space to validate whether it really
+                        * differs from -1.
+                        */
+                       continue;
+               }
                switch (errno) {
-               case EWOULDBLOCK:
+               case EAGAIN:
                        /* Value already changed. */
                        return;
                case EINTR:
                        /* Retry if interrupted by signal. */
-                       break;  /* Get out of switch. */
+                       break;  /* Get out of switch. Check again. */
                default:
                        /* Unexpected error. */
                        urcu_die(errno);
@@ -277,17 +285,25 @@ static void call_rcu_completion_wait(struct call_rcu_completion *completion)
 {
        /* Read completion barrier count before read futex */
        cmm_smp_mb();
-       if (uatomic_read(&completion->futex) != -1)
-               return;
-       while (futex_async(&completion->futex, FUTEX_WAIT, -1,
-                       NULL, NULL, 0)) {
+       while (uatomic_read(&completion->futex) == -1) {
+               if (!futex_async(&completion->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+                       /*
+                        * Prior queued wakeups queued by unrelated code
+                        * using the same address can cause futex wait to
+                        * return 0 even through the futex value is still
+                        * -1 (spurious wakeups). Check the value again
+                        * in user-space to validate whether it really
+                        * differs from -1.
+                        */
+                       continue;
+               }
                switch (errno) {
-               case EWOULDBLOCK:
+               case EAGAIN:
                        /* Value already changed. */
                        return;
                case EINTR:
                        /* Retry if interrupted by signal. */
-                       break;  /* Get out of switch. */
+                       break;  /* Get out of switch. Check again. */
                default:
                        /* Unexpected error. */
                        urcu_die(errno);
@@ -772,9 +788,13 @@ void call_rcu_data_free(struct call_rcu_data *crdp)
                while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0)
                        (void) poll(NULL, 0, 1);
        }
+       call_rcu_lock(&call_rcu_mutex);
        if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) {
-               /* Create default call rcu data if need be */
+               call_rcu_unlock(&call_rcu_mutex);
+               /* Create default call rcu data if need be. */
+               /* CBs queued here will be handed to the default list. */
                (void) get_default_call_rcu_data();
+               call_rcu_lock(&call_rcu_mutex);
                __cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head,
                        &default_call_rcu_data->cbs_tail,
                        &crdp->cbs_head, &crdp->cbs_tail);
@@ -783,7 +803,6 @@ void call_rcu_data_free(struct call_rcu_data *crdp)
                wake_call_rcu_thread(default_call_rcu_data);
        }
 
-       call_rcu_lock(&call_rcu_mutex);
        cds_list_del(&crdp->list);
        call_rcu_unlock(&call_rcu_mutex);
 
@@ -1050,7 +1069,7 @@ end:
 URCU_ATTR_ALIAS(urcu_stringify(urcu_register_rculfhash_atfork))
 void alias_urcu_register_rculfhash_atfork();
 
-void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork)
+void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused)))
 {
        call_rcu_lock(&call_rcu_mutex);
        if (--registered_rculfhash_atfork_refcount)