* half-wait-free/half-blocking queue implementation done by Paul E.
* McKenney.
*
- * Mutual exclusion of __cds_wfcq_* API
- *
- * Unless otherwise stated, the caller must ensure mutual exclusion of
- * queue update operations "dequeue" and "splice" (for source queue).
- * Queue read operations "first" and "next", which are used by
- * "for_each" iterations, need to be protected against concurrent
- * "dequeue" and "splice" (for source queue) by the caller.
- * "enqueue", "splice" (for destination queue), and "empty" are the only
- * operations that can be used without any mutual exclusion.
+ * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * Legend:
+ * [1] cds_wfcq_enqueue
+ * [2] __cds_wfcq_splice (destination queue)
+ * [3] __cds_wfcq_dequeue
+ * [4] __cds_wfcq_splice (source queue)
+ * [5] __cds_wfcq_first
+ * [6] __cds_wfcq_next
+ *
+ * [1] [2] [3] [4] [5] [6]
+ * [1] - - - - - -
+ * [2] - - - - - -
+ * [3] - - X X X X
+ * [4] - - X - X X
+ * [5] - - X X - -
+ * [6] - - X X - -
+ *
* Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
*
* For convenience, cds_wfcq_dequeue_blocking() and
return ___cds_wfcq_append(head, tail, new_tail, new_tail);
}
+/*
+ * ___cds_wfcq_busy_wait: adaptative busy-wait.
+ *
+ * Returns 1 if nonblocking and needs to block, 0 otherwise.
+ */
+static inline bool
+___cds_wfcq_busy_wait(int *attempt, int blocking)
+{
+ if (!blocking)
+ return 1;
+ if (++(*attempt) >= WFCQ_ADAPT_ATTEMPTS) {
+ poll(NULL, 0, WFCQ_WAIT); /* Wait for 10ms */
+ *attempt = 0;
+ } else {
+ caa_cpu_relax();
+ }
+ return 0;
+}
+
/*
* Waiting for enqueuer to complete enqueue and return the next node.
*/
* Adaptative busy-looping waiting for enqueuer to complete enqueue.
*/
while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- if (!blocking)
+ if (___cds_wfcq_busy_wait(&attempt, blocking))
return CDS_WFCQ_WOULDBLOCK;
- if (++attempt >= WFCQ_ADAPT_ATTEMPTS) {
- poll(NULL, 0, WFCQ_WAIT); /* Wait for 10ms */
- attempt = 0;
- } else {
- caa_cpu_relax();
- }
}
return next;
return ___cds_wfcq_dequeue(head, tail, 0);
}
+/*
+ * __cds_wfcq_splice: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue.
+ */
static inline enum cds_wfcq_ret
___cds_wfcq_splice(
struct cds_wfcq_head *dest_q_head,
int blocking)
{
struct cds_wfcq_node *head, *tail;
+ int attempt = 0;
+ /*
+ * Initial emptiness check to speed up cases where queue is
+ * empty: only require loads to check if queue is empty.
+ */
if (_cds_wfcq_empty(src_q_head, src_q_tail))
return CDS_WFCQ_RET_SRC_EMPTY;
- head = ___cds_wfcq_node_sync_next(&src_q_head->node, blocking);
- if (head == CDS_WFCQ_WOULDBLOCK)
- return CDS_WFCQ_RET_WOULDBLOCK;
- _cds_wfcq_node_init(&src_q_head->node);
+ for (;;) {
+ /*
+ * Open-coded _cds_wfcq_empty() by testing result of
+ * uatomic_xchg, as well as tail pointer vs head node
+ * address.
+ */
+ head = uatomic_xchg(&src_q_head->node.next, NULL);
+ if (head)
+ break; /* non-empty */
+ if (CMM_LOAD_SHARED(src_q_tail->p) == &src_q_head->node)
+ return CDS_WFCQ_RET_SRC_EMPTY;
+ if (___cds_wfcq_busy_wait(&attempt, blocking))
+ return CDS_WFCQ_RET_WOULDBLOCK;
+ }
/*
* Memory barrier implied before uatomic_xchg() orders store to
return CDS_WFCQ_RET_DEST_EMPTY;
}
-
/*
* __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
*
* Dequeue all nodes from src_q.
* dest_q must be already initialized.
- * Dequeue/splice/iteration mutual exclusion for src_q should be ensured
- * by the caller.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
* Returns enum cds_wfcq_ret which indicates the state of the src or
* dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
*/
#define CDS_WFCQ_WOULDBLOCK ((void *) -1UL)
enum cds_wfcq_ret {
- CDS_WFCQ_RET_WOULDBLOCK = -1,
+ CDS_WFCQ_RET_WOULDBLOCK = -1,
CDS_WFCQ_RET_DEST_EMPTY = 0,
CDS_WFCQ_RET_DEST_NON_EMPTY = 1,
CDS_WFCQ_RET_SRC_EMPTY = 2,
/*
* Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
*
- * Unless otherwise stated, the caller must ensure mutual exclusion of
- * queue update operations "dequeue" and "splice" (for source queue).
- * Queue read operations "first" and "next", which are used by
- * "for_each" iterations, need to be protected against concurrent
- * "dequeue" and "splice" (for source queue) by the caller.
- * "enqueue", "splice" (for destination queue), and "empty" are the only
- * operations that can be used without any mutual exclusion.
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * Legend:
+ * [1] cds_wfcq_enqueue
+ * [2] __cds_wfcq_splice (destination queue)
+ * [3] __cds_wfcq_dequeue
+ * [4] __cds_wfcq_splice (source queue)
+ * [5] __cds_wfcq_first
+ * [6] __cds_wfcq_next
+ *
+ * [1] [2] [3] [4] [5] [6]
+ * [1] - - - - - -
+ * [2] - - - - - -
+ * [3] - - X X X X
+ * [4] - - X - X X
+ * [5] - - X X - -
+ * [6] - - X X - -
+ *
* Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
*
* For convenience, cds_wfcq_dequeue_blocking() and
*
* Dequeue all nodes from src_q.
* dest_q must be already initialized.
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion for src_q should be ensured
- * by the caller.
- *
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
* Returns enum cds_wfcq_ret which indicates the state of the src or
- * dest queue. Cannot block.
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
*/
extern enum cds_wfcq_ret __cds_wfcq_splice_blocking(
struct cds_wfcq_head *dest_q_head,
projects / urcu.git / commitdiff
