Fix: consumerd: slow metadata push slows down application registration

[lttng-tools.git] / src / common / waiter.c

diff --git a/src/common/waiter.c b/src/common/waiter.c
index c5a2829408215784952d047e272c12682e291c23..e72f53bd46ec31ab7582bda10bcd04ceeaf7f86f 100644 (file)
--- a/src/common/waiter.c
+++ b/src/common/waiter.c
@@ -1,21 +1,9 @@
 /*
- * Copyright (C) 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *               2017 - Jérémie Galarneau <jeremie.galarneau@efficios.com>
+ * Copyright (C) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (C) 2017 Jérémie Galarneau <jeremie.galarneau@efficios.com>
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License, version 2.1 only,
- * as published by the Free Software Foundation.
+ * SPDX-License-Identifier: LGPL-2.1-only
  *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
- * for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * This code is originally adapted from userspace-rcu's urcu-wait.h
  */
 
 #include "waiter.h"
@@ -55,24 +43,38 @@ void lttng_waiter_wait(struct lttng_waiter *waiter)
 {
        unsigned int i;
 
-       DBG("Beginning of waiter wait period");
-       /* Load and test condition before read state */
+       DBG("Beginning of waiter \"wait\" period");
+
+       /* Load and test condition before read state. */
        cmm_smp_rmb();
        for (i = 0; i < WAIT_ATTEMPTS; i++) {
                if (uatomic_read(&waiter->state) != WAITER_WAITING) {
                        goto skip_futex_wait;
                }
+
                caa_cpu_relax();
        }
-       while (futex_noasync(&waiter->state, FUTEX_WAIT, WAITER_WAITING,
-                       NULL, NULL, 0)) {
+
+       while (uatomic_read(&waiter->state) == WAITER_WAITING) {
+               if (!futex_noasync(&waiter->state, FUTEX_WAIT, WAITER_WAITING, NULL, NULL, 0)) {
+                       /*
+                        * Prior queued wakeups queued by unrelated code
+                        * using the same address can cause futex wait to
+                        * return 0 even through the futex value is still
+                        * WAITER_WAITING (spurious wakeups). Check
+                        * the value again in user-space to validate
+                        * whether it really differs from WAITER_WAITING.
+                        */
+                       continue;
+               }
+
                switch (errno) {
-               case EWOULDBLOCK:
+               case EAGAIN:
                        /* Value already changed. */
                        goto skip_futex_wait;
                case EINTR:
                        /* Retry if interrupted by signal. */
-                       break;  /* Get out of switch. */
+                       break;  /* Get out of switch. Check again. */
                default:
                        /* Unexpected error. */
                        PERROR("futex_noasync");
@@ -92,13 +94,16 @@ skip_futex_wait:
                if (uatomic_read(&waiter->state) & WAITER_TEARDOWN) {
                        break;
                }
+
                caa_cpu_relax();
        }
+
        while (!(uatomic_read(&waiter->state) & WAITER_TEARDOWN)) {
                poll(NULL, 0, 10);
        }
+
        assert(uatomic_read(&waiter->state) & WAITER_TEARDOWN);
-       DBG("End of waiter wait period");
+       DBG("End of waiter \"wait\" period");
 }
 
 /*
@@ -107,7 +112,7 @@ skip_futex_wait:
  * it to free this memory when it sees the WAITER_TEARDOWN flag.
  */
 LTTNG_HIDDEN
-void lttng_waiter_wake_up(struct lttng_waiter *waiter)
+void lttng_waiter_wake(struct lttng_waiter *waiter)
 {
        cmm_smp_mb();
        assert(uatomic_read(&waiter->state) == WAITER_WAITING);
@@ -119,6 +124,44 @@ void lttng_waiter_wake_up(struct lttng_waiter *waiter)
                        abort();
                }
        }
+
        /* Allow teardown of struct urcu_wait memory. */
        uatomic_or(&waiter->state, WAITER_TEARDOWN);
 }
+
+
+LTTNG_HIDDEN
+void lttng_wait_queue_init(struct lttng_wait_queue *queue)
+{
+       cds_wfs_init(&queue->stack);
+}
+
+LTTNG_HIDDEN
+void lttng_wait_queue_add(struct lttng_wait_queue *queue,
+               struct lttng_waiter *waiter)
+{
+       (void) cds_wfs_push(&queue->stack, &waiter->wait_queue_node);
+}
+
+LTTNG_HIDDEN
+void lttng_wait_queue_wake_all(struct lttng_wait_queue *queue)
+{
+       struct cds_wfs_head *waiters;
+       struct cds_wfs_node *iter, *iter_n;
+
+       /* Move all waiters from the queue to our local stack. */
+       waiters = __cds_wfs_pop_all(&queue->stack);
+
+       /* Wake all waiters in our stack head. */
+       cds_wfs_for_each_blocking_safe(waiters, iter, iter_n) {
+               struct lttng_waiter *waiter =
+                       container_of(iter, struct lttng_waiter, wait_queue_node);
+
+               /* Don't wake already running threads. */
+               if (waiter->state & WAITER_RUNNING) {
+                       continue;
+               }
+
+               lttng_waiter_wake(waiter);
+       }
+}