#include <sched.h>
#include "compat-getcpu.h"
-#include "urcu/wfcqueue.h"
-#include "urcu-call-rcu.h"
-#include "urcu-pointer.h"
-#include "urcu/list.h"
-#include "urcu/futex.h"
-#include "urcu/tls-compat.h"
-#include "urcu/ref.h"
+#include <urcu/wfcqueue.h>
+#include <urcu/call-rcu.h>
+#include <urcu/pointer.h>
+#include <urcu/list.h>
+#include <urcu/futex.h>
+#include <urcu/tls-compat.h>
+#include <urcu/ref.h>
#include "urcu-die.h"
+#include "urcu-utils.h"
#define SET_AFFINITY_CHECK_PERIOD (1U << 8) /* 256 */
#define SET_AFFINITY_CHECK_PERIOD_MASK (SET_AFFINITY_CHECK_PERIOD - 1)
static struct call_rcu_data *default_call_rcu_data;
+static struct urcu_atfork *registered_rculfhash_atfork;
+static unsigned long registered_rculfhash_atfork_refcount;
+
/*
* If the sched_getcpu() and sysconf(_SC_NPROCESSORS_CONF) calls are
* available, then we can have call_rcu threads assigned to individual
* Losing affinity can be caused by CPU hotunplug/hotplug, or by
* cpuset(7).
*/
-#if HAVE_SCHED_SETAFFINITY
+#ifdef HAVE_SCHED_SETAFFINITY
static
int set_thread_cpu_affinity(struct call_rcu_data *crdp)
{
CPU_ZERO(&mask);
CPU_SET(crdp->cpu_affinity, &mask);
-#if SCHED_SETAFFINITY_ARGS == 2
- ret = sched_setaffinity(0, &mask);
-#else
ret = sched_setaffinity(0, sizeof(mask), &mask);
-#endif
+
/*
* EINVAL is fine: can be caused by hotunplugged CPUs, or by
* cpuset(7). This is why we should always retry if we detect
}
#else
static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((unused)))
{
return 0;
}
return NULL;
return rcu_dereference(pcpu_crdp[cpu]);
}
+URCU_ATTR_ALIAS(urcu_stringify(get_cpu_call_rcu_data))
+struct call_rcu_data *alias_get_cpu_call_rcu_data();
/*
* Return the tid corresponding to the call_rcu thread whose
{
return crdp->tid;
}
+URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_thread))
+pthread_t alias_get_call_rcu_thread();
/*
* Create a call_rcu_data structure (with thread) and return a pointer.
return crdp;
}
+URCU_ATTR_ALIAS(urcu_stringify(create_call_rcu_data))
+struct call_rcu_data *alias_create_call_rcu_data();
struct call_rcu_data *create_call_rcu_data(unsigned long flags,
int cpu_affinity)
{
call_rcu_unlock(&call_rcu_mutex);
return 0;
}
+URCU_ATTR_ALIAS(urcu_stringify(set_cpu_call_rcu_data))
+int alias_set_cpu_call_rcu_data();
/*
* Return a pointer to the default call_rcu_data structure, creating
call_rcu_unlock(&call_rcu_mutex);
return default_call_rcu_data;
}
+URCU_ATTR_ALIAS(urcu_stringify(get_default_call_rcu_data))
+struct call_rcu_data *alias_get_default_call_rcu_data();
/*
* Return the call_rcu_data structure that applies to the currently
return get_default_call_rcu_data();
}
+URCU_ATTR_ALIAS(urcu_stringify(get_call_rcu_data))
+struct call_rcu_data *alias_get_call_rcu_data();
/*
* Return a pointer to this task's call_rcu_data if there is one.
{
return URCU_TLS(thread_call_rcu_data);
}
+URCU_ATTR_ALIAS(urcu_stringify(get_thread_call_rcu_data))
+struct call_rcu_data *alias_get_thread_call_rcu_data();
/*
* Set this task's call_rcu_data structure as specified, regardless
{
URCU_TLS(thread_call_rcu_data) = crdp;
}
+URCU_ATTR_ALIAS(urcu_stringify(set_thread_call_rcu_data))
+void alias_set_thread_call_rcu_data();
/*
* Create a separate call_rcu thread for each CPU. This does not
}
return 0;
}
+URCU_ATTR_ALIAS(urcu_stringify(create_all_cpu_call_rcu_data))
+int alias_create_all_cpu_call_rcu_data();
/*
* Wake up the call_rcu thread corresponding to the specified
_call_rcu(head, func, crdp);
_rcu_read_unlock();
}
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu)) void alias_call_rcu();
/*
* Free up the specified call_rcu_data structure, terminating the
while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0)
(void) poll(NULL, 0, 1);
}
+ call_rcu_lock(&call_rcu_mutex);
if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) {
- /* Create default call rcu data if need be */
+ call_rcu_unlock(&call_rcu_mutex);
+ /* Create default call rcu data if need be. */
+ /* CBs queued here will be handed to the default list. */
(void) get_default_call_rcu_data();
+ call_rcu_lock(&call_rcu_mutex);
__cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head,
&default_call_rcu_data->cbs_tail,
&crdp->cbs_head, &crdp->cbs_tail);
wake_call_rcu_thread(default_call_rcu_data);
}
- call_rcu_lock(&call_rcu_mutex);
cds_list_del(&crdp->list);
call_rcu_unlock(&call_rcu_mutex);
free(crdp);
}
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_data_free))
+void alias_call_rcu_data_free();
/*
* Clean up all the per-CPU call_rcu threads.
}
free(crdp);
}
+#ifdef RCU_QSBR
+/* ABI6 has a non-namespaced free_all_cpu_call_rcu_data for qsbr */
+#undef free_all_cpu_call_rcu_data
+URCU_ATTR_ALIAS("urcu_qsbr_free_all_cpu_call_rcu_data")
+void free_all_cpu_call_rcu_data();
+#define free_all_cpu_call_rcu_data urcu_qsbr_free_all_cpu_call_rcu_data
+#else
+URCU_ATTR_ALIAS(urcu_stringify(free_all_cpu_call_rcu_data))
+void alias_free_all_cpu_call_rcu_data();
+#endif
static
void free_completion(struct urcu_ref *ref)
if (was_online)
rcu_thread_online();
}
+URCU_ATTR_ALIAS(urcu_stringify(rcu_barrier))
+void alias_rcu_barrier();
/*
* Acquire the call_rcu_mutex in order to ensure that the child sees
void call_rcu_before_fork(void)
{
struct call_rcu_data *crdp;
+ struct urcu_atfork *atfork;
call_rcu_lock(&call_rcu_mutex);
+ atfork = registered_rculfhash_atfork;
+ if (atfork)
+ atfork->before_fork(atfork->priv);
+
cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSE);
cmm_smp_mb__after_uatomic_or();
(void) poll(NULL, 0, 1);
}
}
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_before_fork))
+void alias_call_rcu_before_fork();
/*
* Clean up call_rcu data structures in the parent of a successful fork()
void call_rcu_after_fork_parent(void)
{
struct call_rcu_data *crdp;
+ struct urcu_atfork *atfork;
cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSE);
while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) != 0)
(void) poll(NULL, 0, 1);
}
+ atfork = registered_rculfhash_atfork;
+ if (atfork)
+ atfork->after_fork_parent(atfork->priv);
call_rcu_unlock(&call_rcu_mutex);
}
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_parent))
+void alias_call_rcu_after_fork_parent();
/*
* Clean up call_rcu data structures in the child of a successful fork()
void call_rcu_after_fork_child(void)
{
struct call_rcu_data *crdp, *next;
+ struct urcu_atfork *atfork;
/* Release the mutex. */
call_rcu_unlock(&call_rcu_mutex);
+ atfork = registered_rculfhash_atfork;
+ if (atfork)
+ atfork->after_fork_child(atfork->priv);
+
/* Do nothing when call_rcu() has not been used */
if (cds_list_empty(&call_rcu_data_list))
return;
call_rcu_data_free(crdp);
}
}
+URCU_ATTR_ALIAS(urcu_stringify(call_rcu_after_fork_child))
+void alias_call_rcu_after_fork_child();
+
+void urcu_register_rculfhash_atfork(struct urcu_atfork *atfork)
+{
+ call_rcu_lock(&call_rcu_mutex);
+ if (registered_rculfhash_atfork_refcount++)
+ goto end;
+ registered_rculfhash_atfork = atfork;
+end:
+ call_rcu_unlock(&call_rcu_mutex);
+}
+URCU_ATTR_ALIAS(urcu_stringify(urcu_register_rculfhash_atfork))
+void alias_urcu_register_rculfhash_atfork();
+
+void urcu_unregister_rculfhash_atfork(struct urcu_atfork *atfork __attribute__((unused)))
+{
+ call_rcu_lock(&call_rcu_mutex);
+ if (--registered_rculfhash_atfork_refcount)
+ goto end;
+ registered_rculfhash_atfork = NULL;
+end:
+ call_rcu_unlock(&call_rcu_mutex);
+}
+URCU_ATTR_ALIAS(urcu_stringify(urcu_unregister_rculfhash_atfork))
+void alias_urcu_unregister_rculfhash_atfork();
projects / urcu.git / blobdiff