X-Git-Url: https://git.lttng.org/?p=urcu.git;a=blobdiff_plain;f=urcu-defer-impl.h;h=f65e41086022f2613bdaf251611a378afb6e5a3d;hp=34d99c92997641323175c909f13f793fa27df33d;hb=bd252a04bbbb163aa4d8864b1e1e5a3a4d9d0892;hpb=2ef1bfb2cb9abb657e077814e272801e197a3c47 diff --git a/urcu-defer-impl.h b/urcu-defer-impl.h index 34d99c9..f65e410 100644 --- a/urcu-defer-impl.h +++ b/urcu-defer-impl.h @@ -48,6 +48,7 @@ #include #include #include +#include /* * Number of entries in the per-thread defer queue. Must be power of 2. @@ -130,7 +131,7 @@ static int32_t defer_thread_stop; * Written to only by each individual deferer. Read by both the deferer and * the reclamation tread. */ -static struct defer_queue __thread defer_queue; +static DEFINE_URCU_TLS(struct defer_queue, defer_queue); static CDS_LIST_HEAD(registry_defer); static pthread_t tid_defer; @@ -161,7 +162,7 @@ static void mutex_lock_defer(pthread_mutex_t *mutex) */ static void wake_up_defer(void) { - if (unlikely(uatomic_read(&defer_thread_futex) == -1)) { + if (caa_unlikely(uatomic_read(&defer_thread_futex) == -1)) { uatomic_set(&defer_thread_futex, 0); futex_noasync(&defer_thread_futex, FUTEX_WAKE, 1, NULL, NULL, 0); @@ -225,11 +226,11 @@ static void rcu_defer_barrier_queue(struct defer_queue *queue, for (i = queue->tail; i != head;) { cmm_smp_rmb(); /* read head before q[]. */ p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); - if (unlikely(DQ_IS_FCT_BIT(p))) { + if (caa_unlikely(DQ_IS_FCT_BIT(p))) { DQ_CLEAR_FCT_BIT(p); queue->last_fct_out = p; p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); - } else if (unlikely(p == DQ_FCT_MARK)) { + } else if (caa_unlikely(p == DQ_FCT_MARK)) { p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); queue->last_fct_out = p; p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); @@ -245,12 +246,12 @@ static void _rcu_defer_barrier_thread(void) { unsigned long head, num_items; - head = defer_queue.head; - num_items = head - defer_queue.tail; - if (unlikely(!num_items)) + head = URCU_TLS(defer_queue).head; + num_items = head - URCU_TLS(defer_queue).tail; + if (caa_unlikely(!num_items)) return; synchronize_rcu(); - rcu_defer_barrier_queue(&defer_queue, head); + rcu_defer_barrier_queue(&URCU_TLS(defer_queue), head); } void rcu_defer_barrier_thread(void) @@ -286,7 +287,7 @@ void rcu_defer_barrier(void) index->last_head = CMM_LOAD_SHARED(index->head); num_items += index->last_head - index->tail; } - if (likely(!num_items)) { + if (caa_likely(!num_items)) { /* * We skip the grace period because there are no queued * callbacks to execute. @@ -311,17 +312,17 @@ void _defer_rcu(void (*fct)(void *p), void *p) * Head is only modified by ourself. Tail can be modified by reclamation * thread. */ - head = defer_queue.head; - tail = CMM_LOAD_SHARED(defer_queue.tail); + head = URCU_TLS(defer_queue).head; + tail = CMM_LOAD_SHARED(URCU_TLS(defer_queue).tail); /* * If queue is full, or reached threshold. Empty queue ourself. * Worse-case: must allow 2 supplementary entries for fct pointer. */ - if (unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) { + if (caa_unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) { assert(head - tail <= DEFER_QUEUE_SIZE); rcu_defer_barrier_thread(); - assert(head - CMM_LOAD_SHARED(defer_queue.tail) == 0); + assert(head - CMM_LOAD_SHARED(URCU_TLS(defer_queue).tail) == 0); } /* @@ -340,25 +341,25 @@ void _defer_rcu(void (*fct)(void *p), void *p) * Decode: see the comments before 'struct defer_queue' * or the code in rcu_defer_barrier_queue(). */ - if (unlikely(defer_queue.last_fct_in != fct + if (caa_unlikely(URCU_TLS(defer_queue).last_fct_in != fct || DQ_IS_FCT_BIT(p) || p == DQ_FCT_MARK)) { - defer_queue.last_fct_in = fct; - if (unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) { - _CMM_STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], + URCU_TLS(defer_queue).last_fct_in = fct; + if (caa_unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) { + _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK], DQ_FCT_MARK); - _CMM_STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], + _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK], fct); } else { DQ_SET_FCT_BIT(fct); - _CMM_STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], + _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK], fct); } } - _CMM_STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], p); + _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK], p); cmm_smp_wmb(); /* Publish new pointer before head */ /* Write q[] before head. */ - CMM_STORE_SHARED(defer_queue.head, head); + CMM_STORE_SHARED(URCU_TLS(defer_queue).head, head); cmm_smp_mb(); /* Write queue head before read futex */ /* * Wake-up any waiting defer thread. @@ -422,16 +423,16 @@ int rcu_defer_register_thread(void) { int was_empty; - assert(defer_queue.last_head == 0); - assert(defer_queue.q == NULL); - defer_queue.q = malloc(sizeof(void *) * DEFER_QUEUE_SIZE); - if (!defer_queue.q) + assert(URCU_TLS(defer_queue).last_head == 0); + assert(URCU_TLS(defer_queue).q == NULL); + URCU_TLS(defer_queue).q = malloc(sizeof(void *) * DEFER_QUEUE_SIZE); + if (!URCU_TLS(defer_queue).q) return -ENOMEM; mutex_lock_defer(&defer_thread_mutex); mutex_lock_defer(&rcu_defer_mutex); was_empty = cds_list_empty(®istry_defer); - cds_list_add(&defer_queue.list, ®istry_defer); + cds_list_add(&URCU_TLS(defer_queue).list, ®istry_defer); mutex_unlock(&rcu_defer_mutex); if (was_empty) @@ -446,10 +447,10 @@ void rcu_defer_unregister_thread(void) mutex_lock_defer(&defer_thread_mutex); mutex_lock_defer(&rcu_defer_mutex); - cds_list_del(&defer_queue.list); + cds_list_del(&URCU_TLS(defer_queue).list); _rcu_defer_barrier_thread(); - free(defer_queue.q); - defer_queue.q = NULL; + free(URCU_TLS(defer_queue).q); + URCU_TLS(defer_queue).q = NULL; is_empty = cds_list_empty(®istry_defer); mutex_unlock(&rcu_defer_mutex);