X-Git-Url: https://git.lttng.org/?a=blobdiff_plain;f=src%2Furcu-call-rcu-impl.h;h=9f85d55b403c5fa1f882947af021c6cae460d712;hb=72886af724b546cd14c682af93c88bfe59998015;hp=56fe943ac8d1ab8b9b5d260efca4d9e1ed8d7728;hpb=6fd172f599e8d798e68974a786dd930d876f182e;p=urcu.git diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h index 56fe943..9f85d55 100644 --- a/src/urcu-call-rcu-impl.h +++ b/src/urcu-call-rcu-impl.h @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -35,14 +34,17 @@ #include #include "compat-getcpu.h" -#include "urcu/wfcqueue.h" -#include "urcu-call-rcu.h" -#include "urcu-pointer.h" -#include "urcu/list.h" -#include "urcu/futex.h" -#include "urcu/tls-compat.h" -#include "urcu/ref.h" +#include +#include +#include +#include +#include +#include +#include +#include #include "urcu-die.h" +#include "urcu-utils.h" +#include "compat-smp.h" #define SET_AFFINITY_CHECK_PERIOD (1U << 8) /* 256 */ #define SET_AFFINITY_CHECK_PERIOD_MASK (SET_AFFINITY_CHECK_PERIOD - 1) @@ -88,7 +90,7 @@ static CDS_LIST_HEAD(call_rcu_data_list); /* Link a thread using call_rcu() to its call_rcu thread. */ -static DEFINE_URCU_TLS_IE(struct call_rcu_data *, thread_call_rcu_data); +static DEFINE_URCU_TLS(struct call_rcu_data *, thread_call_rcu_data); /* * Guard call_rcu thread creation and atfork handlers. @@ -119,11 +121,11 @@ static unsigned long registered_rculfhash_atfork_refcount; */ static struct call_rcu_data **per_cpu_call_rcu_data; -static long maxcpus; +static long cpus_array_len; -static void maxcpus_reset(void) +static void cpus_array_len_reset(void) { - maxcpus = 0; + cpus_array_len = 0; } /* Allocate the array if it has not already been allocated. */ @@ -133,15 +135,15 @@ static void alloc_cpu_call_rcu_data(void) struct call_rcu_data **p; static int warned = 0; - if (maxcpus != 0) + if (cpus_array_len != 0) return; - maxcpus = sysconf(_SC_NPROCESSORS_CONF); - if (maxcpus <= 0) { + cpus_array_len = get_possible_cpus_array_len(); + if (cpus_array_len <= 0) { return; } - p = malloc(maxcpus * sizeof(*per_cpu_call_rcu_data)); + p = malloc(cpus_array_len * sizeof(*per_cpu_call_rcu_data)); if (p != NULL) { - memset(p, '\0', maxcpus * sizeof(*per_cpu_call_rcu_data)); + memset(p, '\0', cpus_array_len * sizeof(*per_cpu_call_rcu_data)); rcu_set_pointer(&per_cpu_call_rcu_data, p); } else { if (!warned) { @@ -159,9 +161,9 @@ static void alloc_cpu_call_rcu_data(void) * constant. */ static struct call_rcu_data **per_cpu_call_rcu_data = NULL; -static const long maxcpus = -1; +static const long cpus_array_len = -1; -static void maxcpus_reset(void) +static void cpus_array_len_reset(void) { } @@ -198,7 +200,7 @@ static void call_rcu_unlock(pthread_mutex_t *pmp) * Losing affinity can be caused by CPU hotunplug/hotplug, or by * cpuset(7). */ -#if HAVE_SCHED_SETAFFINITY +#ifdef HAVE_SCHED_SETAFFINITY static int set_thread_cpu_affinity(struct call_rcu_data *crdp) { @@ -214,11 +216,8 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp) CPU_ZERO(&mask); CPU_SET(crdp->cpu_affinity, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - ret = sched_setaffinity(0, &mask); -#else ret = sched_setaffinity(0, sizeof(mask), &mask); -#endif + /* * EINVAL is fine: can be caused by hotunplugged CPUs, or by * cpuset(7). This is why we should always retry if we detect @@ -232,7 +231,7 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp) } #else static -int set_thread_cpu_affinity(struct call_rcu_data *crdp) +int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused))) { return 0; } @@ -242,17 +241,25 @@ static void call_rcu_wait(struct call_rcu_data *crdp) { /* Read call_rcu list before read futex */ cmm_smp_mb(); - if (uatomic_read(&crdp->futex) != -1) - return; - while (futex_async(&crdp->futex, FUTEX_WAIT, -1, - NULL, NULL, 0)) { + while (uatomic_read(&crdp->futex) == -1) { + if (!futex_async(&crdp->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) { + /* + * Prior queued wakeups queued by unrelated code + * using the same address can cause futex wait to + * return 0 even through the futex value is still + * -1 (spurious wakeups). Check the value again + * in user-space to validate whether it really + * differs from -1. + */ + continue; + } switch (errno) { - case EWOULDBLOCK: + case EAGAIN: /* Value already changed. */ return; case EINTR: /* Retry if interrupted by signal. */ - break; /* Get out of switch. */ + break; /* Get out of switch. Check again. */ default: /* Unexpected error. */ urcu_die(errno); @@ -276,17 +283,25 @@ static void call_rcu_completion_wait(struct call_rcu_completion *completion) { /* Read completion barrier count before read futex */ cmm_smp_mb(); - if (uatomic_read(&completion->futex) != -1) - return; - while (futex_async(&completion->futex, FUTEX_WAIT, -1, - NULL, NULL, 0)) { + while (uatomic_read(&completion->futex) == -1) { + if (!futex_async(&completion->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) { + /* + * Prior queued wakeups queued by unrelated code + * using the same address can cause futex wait to + * return 0 even through the futex value is still + * -1 (spurious wakeups). Check the value again + * in user-space to validate whether it really + * differs from -1. + */ + continue; + } switch (errno) { - case EWOULDBLOCK: + case EAGAIN: /* Value already changed. */ return; case EINTR: /* Retry if interrupted by signal. */ - break; /* Get out of switch. */ + break; /* Get out of switch. Check again. */ default: /* Unexpected error. */ urcu_die(errno); @@ -357,8 +372,8 @@ static void *call_rcu_thread(void *arg) cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail); splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head, &cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail); - assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK); - assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY); + urcu_posix_assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK); + urcu_posix_assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY); if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) { synchronize_rcu(); cbcount = 0; @@ -419,6 +434,7 @@ static void call_rcu_data_init(struct call_rcu_data **crdpp, { struct call_rcu_data *crdp; int ret; + sigset_t newmask, oldmask; crdp = malloc(sizeof(*crdp)); if (crdp == NULL) @@ -433,9 +449,18 @@ static void call_rcu_data_init(struct call_rcu_data **crdpp, crdp->gp_count = 0; cmm_smp_mb(); /* Structure initialized before pointer is planted. */ *crdpp = crdp; + + ret = sigfillset(&newmask); + urcu_posix_assert(!ret); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); + urcu_posix_assert(!ret); + ret = pthread_create(&crdp->tid, NULL, call_rcu_thread, crdp); if (ret) urcu_die(ret); + + ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); + urcu_posix_assert(!ret); } /* @@ -456,11 +481,11 @@ struct call_rcu_data *get_cpu_call_rcu_data(int cpu) pcpu_crdp = rcu_dereference(per_cpu_call_rcu_data); if (pcpu_crdp == NULL) return NULL; - if (!warned && maxcpus > 0 && (cpu < 0 || maxcpus <= cpu)) { + if (!warned && cpus_array_len > 0 && (cpu < 0 || cpus_array_len <= cpu)) { fprintf(stderr, "[error] liburcu: get CPU # out of range\n"); warned = 1; } - if (cpu < 0 || maxcpus <= cpu) + if (cpu < 0 || cpus_array_len <= cpu) return NULL; return rcu_dereference(pcpu_crdp[cpu]); } @@ -518,7 +543,7 @@ int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp) call_rcu_lock(&call_rcu_mutex); alloc_cpu_call_rcu_data(); - if (cpu < 0 || maxcpus <= cpu) { + if (cpu < 0 || cpus_array_len <= cpu) { if (!warned) { fprintf(stderr, "[error] liburcu: set CPU # out of range\n"); warned = 1; @@ -583,7 +608,7 @@ struct call_rcu_data *get_call_rcu_data(void) if (URCU_TLS(thread_call_rcu_data) != NULL) return URCU_TLS(thread_call_rcu_data); - if (maxcpus > 0) { + if (cpus_array_len > 0) { crd = get_cpu_call_rcu_data(urcu_sched_getcpu()); if (crd) return crd; @@ -634,7 +659,7 @@ int create_all_cpu_call_rcu_data(unsigned long flags) call_rcu_lock(&call_rcu_mutex); alloc_cpu_call_rcu_data(); call_rcu_unlock(&call_rcu_mutex); - if (maxcpus <= 0) { + if (cpus_array_len <= 0) { errno = EINVAL; return -EINVAL; } @@ -642,7 +667,7 @@ int create_all_cpu_call_rcu_data(unsigned long flags) errno = ENOMEM; return -ENOMEM; } - for (i = 0; i < maxcpus; i++) { + for (i = 0; i < cpus_array_len; i++) { call_rcu_lock(&call_rcu_mutex); if (get_cpu_call_rcu_data(i)) { call_rcu_unlock(&call_rcu_mutex); @@ -752,9 +777,13 @@ void call_rcu_data_free(struct call_rcu_data *crdp) while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0) (void) poll(NULL, 0, 1); } + call_rcu_lock(&call_rcu_mutex); if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) { - /* Create default call rcu data if need be */ + call_rcu_unlock(&call_rcu_mutex); + /* Create default call rcu data if need be. */ + /* CBs queued here will be handed to the default list. */ (void) get_default_call_rcu_data(); + call_rcu_lock(&call_rcu_mutex); __cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head, &default_call_rcu_data->cbs_tail, &crdp->cbs_head, &crdp->cbs_tail); @@ -763,7 +792,6 @@ void call_rcu_data_free(struct call_rcu_data *crdp) wake_call_rcu_thread(default_call_rcu_data); } - call_rcu_lock(&call_rcu_mutex); cds_list_del(&crdp->list); call_rcu_unlock(&call_rcu_mutex); @@ -779,10 +807,10 @@ void free_all_cpu_call_rcu_data(void) struct call_rcu_data **crdp; static int warned = 0; - if (maxcpus <= 0) + if (cpus_array_len <= 0) return; - crdp = malloc(sizeof(*crdp) * maxcpus); + crdp = malloc(sizeof(*crdp) * cpus_array_len); if (!crdp) { if (!warned) { fprintf(stderr, "[error] liburcu: unable to allocate per-CPU pointer array\n"); @@ -791,7 +819,7 @@ void free_all_cpu_call_rcu_data(void) return; } - for (cpu = 0; cpu < maxcpus; cpu++) { + for (cpu = 0; cpu < cpus_array_len; cpu++) { crdp[cpu] = get_cpu_call_rcu_data(cpu); if (crdp[cpu] == NULL) continue; @@ -802,7 +830,7 @@ void free_all_cpu_call_rcu_data(void) * call_rcu_data to become quiescent. */ synchronize_rcu(); - for (cpu = 0; cpu < maxcpus; cpu++) { + for (cpu = 0; cpu < cpus_array_len; cpu++) { if (crdp[cpu] == NULL) continue; call_rcu_data_free(crdp[cpu]); @@ -980,7 +1008,7 @@ void call_rcu_after_fork_child(void) (void)get_default_call_rcu_data(); /* Cleanup call_rcu_data pointers before use */ - maxcpus_reset(); + cpus_array_len_reset(); free(per_cpu_call_rcu_data); rcu_set_pointer(&per_cpu_call_rcu_data, NULL); URCU_TLS(thread_call_rcu_data) = NULL; @@ -1008,7 +1036,7 @@ end: call_rcu_unlock(&call_rcu_mutex); } -void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork) +void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused))) { call_rcu_lock(&call_rcu_mutex); if (--registered_rculfhash_atfork_refcount)