X-Git-Url: https://git.lttng.org/?a=blobdiff_plain;f=urcu-call-rcu-impl.h;h=6580397b1ee42b04a8e48b5c29a76220ea39b6d0;hb=ae25b7e24f130728adfac25df56df8c6a220c719;hp=dca98e4ae073d45f93d8665235b715661f16c0ad;hpb=b9f893b69fbc31baea418794938f4eb74cc4923a;p=urcu.git

diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h
index dca98e4..6580397 100644
--- a/urcu-call-rcu-impl.h
+++ b/urcu-call-rcu-impl.h
@@ -48,15 +48,14 @@
 
 struct call_rcu_data {
 	/*
-	 * Align the tail on cache line size to eliminate false-sharing
-	 * with head. Small note, however: the "qlen" field, kept for
-	 * debugging, will cause false-sharing between enqueue and
-	 * dequeue.
+	 * We do not align head on a different cache-line than tail
+	 * mainly because call_rcu callback-invocation threads use
+	 * batching ("splice") to get an entire list of callbacks, which
+	 * effectively empties the queue, and requires to touch the tail
+	 * anyway.
 	 */
 	struct cds_wfcq_tail cbs_tail;
-	/* Alignment on cache line size will add padding here */
-
-	struct cds_wfcq_head __attribute__((aligned(CAA_CACHE_LINE_SIZE))) cbs_head;
+	struct cds_wfcq_head cbs_head;
 	unsigned long flags;
 	int32_t futex;
 	unsigned long qlen; /* maintained for debugging. */
@@ -70,7 +69,7 @@ struct call_rcu_data {
  * Protected by call_rcu_mutex.
  */
 
-CDS_LIST_HEAD(call_rcu_data_list);
+static CDS_LIST_HEAD(call_rcu_data_list);
 
 /* Link a thread using call_rcu() to its call_rcu thread. */
 
@@ -253,11 +252,14 @@ static void *call_rcu_thread(void *arg)
 		struct cds_wfcq_head cbs_tmp_head;
 		struct cds_wfcq_tail cbs_tmp_tail;
 		struct cds_wfcq_node *cbs, *cbs_tmp_n;
+		enum cds_wfcq_ret splice_ret;
 
 		cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
-		__cds_wfcq_splice_blocking(&cbs_tmp_head, &cbs_tmp_tail,
-			&crdp->cbs_head, &crdp->cbs_tail);
-		if (!cds_wfcq_empty(&cbs_tmp_head, &cbs_tmp_tail)) {
+		splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
+			&cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
+		assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
+		assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
+		if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
 			synchronize_rcu();
 			cbcount = 0;
 			__cds_wfcq_for_each_blocking_safe(&cbs_tmp_head,
@@ -627,6 +629,10 @@ void call_rcu(struct rcu_head *head,
  * The caller must wait for a grace-period to pass between return from
  * set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
  * previous call rcu data as argument.
+ *
+ * Note: introducing __cds_wfcq_splice_blocking() in this function fixed
+ * a list corruption bug in the 0.7.x series. The equivalent fix
+ * appeared in 0.6.8 for the stable-0.6 branch.
  */
 void call_rcu_data_free(struct call_rcu_data *crdp)
 {