struct call_rcu_data {
/*
- * Align the tail on cache line size to eliminate false-sharing
- * with head.
+ * We do not align head on a different cache-line than tail
+ * mainly because call_rcu callback-invocation threads use
+ * batching ("splice") to get an entire list of callbacks, which
+ * effectively empties the queue, and requires to touch the tail
+ * anyway.
*/
- struct cds_wfcq_tail __attribute__((aligned(CAA_CACHE_LINE_SIZE))) cbs_tail;
- /* Alignment on cache line size will add padding here */
-
+ struct cds_wfcq_tail cbs_tail;
struct cds_wfcq_head cbs_head;
unsigned long flags;
int32_t futex;
* Protected by call_rcu_mutex.
*/
-CDS_LIST_HEAD(call_rcu_data_list);
+static CDS_LIST_HEAD(call_rcu_data_list);
/* Link a thread using call_rcu() to its call_rcu thread. */
struct cds_wfcq_head cbs_tmp_head;
struct cds_wfcq_tail cbs_tmp_tail;
struct cds_wfcq_node *cbs, *cbs_tmp_n;
+ enum cds_wfcq_ret splice_ret;
cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
- __cds_wfcq_splice_blocking(&cbs_tmp_head, &cbs_tmp_tail,
- &crdp->cbs_head, &crdp->cbs_tail);
- if (!cds_wfcq_empty(&cbs_tmp_head, &cbs_tmp_tail)) {
+ splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
+ &cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
+ assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
+ assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
+ if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
synchronize_rcu();
cbcount = 0;
__cds_wfcq_for_each_blocking_safe(&cbs_tmp_head,
* The caller must wait for a grace-period to pass between return from
* set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
* previous call rcu data as argument.
+ *
+ * Note: introducing __cds_wfcq_splice_blocking() in this function fixed
+ * a list corruption bug in the 0.7.x series. The equivalent fix
+ * appeared in 0.6.8 for the stable-0.6 branch.
*/
void call_rcu_data_free(struct call_rcu_data *crdp)
{