X-Git-Url: https://git.lttng.org/?a=blobdiff_plain;f=libringbuffer%2Fbackend_internal.h;h=36d53dfd531357037501a539cd8c991c9d2ca4d9;hb=aae88c703374f4b1fbb8a5e7e95591bf8ce3e837;hp=182ac97c6d72d3163d2c88be9f7a312d640f9473;hpb=a6352fd40a2090fd883a6c369144bf405c9e9ec4;p=lttng-ust.git diff --git a/libringbuffer/backend_internal.h b/libringbuffer/backend_internal.h index 182ac97c..36d53dfd 100644 --- a/libringbuffer/backend_internal.h +++ b/libringbuffer/backend_internal.h @@ -14,7 +14,7 @@ #include #include -#include "config.h" +#include #include "backend_types.h" #include "frontend_types.h" #include "shm.h" @@ -23,25 +23,28 @@ /* Ring buffer and channel backend create/free */ -int lib_ring_buffer_backend_create(struct lib_ring_buffer_backend *bufb, +int lib_ring_buffer_backend_create(struct lttng_ust_lib_ring_buffer_backend *bufb, struct channel_backend *chan, int cpu, - struct shm_header *shm_header); + struct lttng_ust_shm_handle *handle, + struct shm_object *shmobj); void channel_backend_unregister_notifiers(struct channel_backend *chanb); -void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend *bufb); +void lib_ring_buffer_backend_free(struct lttng_ust_lib_ring_buffer_backend *bufb); int channel_backend_init(struct channel_backend *chanb, const char *name, - const struct lib_ring_buffer_config *config, - void *priv, size_t subbuf_size, - size_t num_subbuf, struct shm_header *shm_header); -void channel_backend_free(struct channel_backend *chanb); - -void lib_ring_buffer_backend_reset(struct lib_ring_buffer_backend *bufb); + const struct lttng_ust_lib_ring_buffer_config *config, + size_t subbuf_size, + size_t num_subbuf, struct lttng_ust_shm_handle *handle); +void channel_backend_free(struct channel_backend *chanb, + struct lttng_ust_shm_handle *handle); + +void lib_ring_buffer_backend_reset(struct lttng_ust_lib_ring_buffer_backend *bufb, + struct lttng_ust_shm_handle *handle); void channel_backend_reset(struct channel_backend *chanb); int lib_ring_buffer_backend_init(void); void lib_ring_buffer_backend_exit(void); -extern void _lib_ring_buffer_write(struct lib_ring_buffer_backend *bufb, +extern void _lib_ring_buffer_write(struct lttng_ust_lib_ring_buffer_backend *bufb, size_t offset, const void *src, size_t len, ssize_t pagecpy); @@ -83,7 +86,7 @@ extern void _lib_ring_buffer_write(struct lib_ring_buffer_backend *bufb, * mode). */ static inline -unsigned long subbuffer_id(const struct lib_ring_buffer_config *config, +unsigned long subbuffer_id(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long offset, unsigned long noref, unsigned long index) { @@ -100,14 +103,14 @@ unsigned long subbuffer_id(const struct lib_ring_buffer_config *config, * bits are identical, else 0. */ static inline -int subbuffer_id_compare_offset(const struct lib_ring_buffer_config *config, +int subbuffer_id_compare_offset(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long id, unsigned long offset) { return (id & SB_ID_OFFSET_MASK) == (offset << SB_ID_OFFSET_SHIFT); } static inline -unsigned long subbuffer_id_get_index(const struct lib_ring_buffer_config *config, +unsigned long subbuffer_id_get_index(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long id) { if (config->mode == RING_BUFFER_OVERWRITE) @@ -117,7 +120,7 @@ unsigned long subbuffer_id_get_index(const struct lib_ring_buffer_config *config } static inline -unsigned long subbuffer_id_is_noref(const struct lib_ring_buffer_config *config, +unsigned long subbuffer_id_is_noref(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long id) { if (config->mode == RING_BUFFER_OVERWRITE) @@ -131,7 +134,7 @@ unsigned long subbuffer_id_is_noref(const struct lib_ring_buffer_config *config, * needed. */ static inline -void subbuffer_id_set_noref(const struct lib_ring_buffer_config *config, +void subbuffer_id_set_noref(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long *id) { if (config->mode == RING_BUFFER_OVERWRITE) @@ -139,7 +142,7 @@ void subbuffer_id_set_noref(const struct lib_ring_buffer_config *config, } static inline -void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config, +void subbuffer_id_set_noref_offset(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long *id, unsigned long offset) { unsigned long tmp; @@ -156,7 +159,7 @@ void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config, /* No volatile access, since already used locally */ static inline -void subbuffer_id_clear_noref(const struct lib_ring_buffer_config *config, +void subbuffer_id_clear_noref(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long *id) { if (config->mode == RING_BUFFER_OVERWRITE) @@ -171,7 +174,7 @@ void subbuffer_id_clear_noref(const struct lib_ring_buffer_config *config, * -EPERM on failure. */ static inline -int subbuffer_id_check_index(const struct lib_ring_buffer_config *config, +int subbuffer_id_check_index(const struct lttng_ust_lib_ring_buffer_config *config, unsigned long num_subbuf) { if (config->mode == RING_BUFFER_OVERWRITE) @@ -181,14 +184,14 @@ int subbuffer_id_check_index(const struct lib_ring_buffer_config *config, } static inline -void subbuffer_count_record(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) +void subbuffer_count_record(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + unsigned long idx, struct lttng_ust_shm_handle *handle) { unsigned long sb_bindex; - sb_bindex = subbuffer_id_get_index(config, shmp(bufb->buf_wsb)[idx].id); - v_inc(config, &shmp(bufb->array)[sb_bindex]->records_commit); + sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id); + v_inc(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_commit); } /* @@ -196,29 +199,31 @@ void subbuffer_count_record(const struct lib_ring_buffer_config *config, * perform the decrement atomically. */ static inline -void subbuffer_consume_record(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb) +void subbuffer_consume_record(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + struct lttng_ust_shm_handle *handle) { unsigned long sb_bindex; sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id); - CHAN_WARN_ON(bufb->chan, - !v_read(config, &shmp(bufb->array)[sb_bindex]->records_unread)); + CHAN_WARN_ON(shmp(handle, bufb->chan), + !v_read(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_unread)); /* Non-atomic decrement protected by exclusive subbuffer access */ - _v_dec(config, &shmp(bufb->array)[sb_bindex]->records_unread); + _v_dec(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_unread); v_inc(config, &bufb->records_read); } static inline unsigned long subbuffer_get_records_count( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) + const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + unsigned long idx, + struct lttng_ust_shm_handle *handle) { unsigned long sb_bindex; - sb_bindex = subbuffer_id_get_index(config, shmp(bufb->buf_wsb)[idx].id); - return v_read(config, &shmp(bufb->array)[sb_bindex]->records_commit); + sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id); + return v_read(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_commit); } /* @@ -229,62 +234,66 @@ unsigned long subbuffer_get_records_count( */ static inline unsigned long subbuffer_count_records_overrun( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) + const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + unsigned long idx, + struct lttng_ust_shm_handle *handle) { - struct lib_ring_buffer_backend_pages *pages; + struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages; unsigned long overruns, sb_bindex; - sb_bindex = subbuffer_id_get_index(config, shmp(bufb->buf_wsb)[idx].id); - pages = shmp(bufb->array)[sb_bindex]; - overruns = v_read(config, &pages->records_unread); - v_set(config, &pages->records_unread, - v_read(config, &pages->records_commit)); - v_set(config, &pages->records_commit, 0); + sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id); + pages = shmp_index(handle, bufb->array, sb_bindex); + overruns = v_read(config, &shmp(handle, pages->shmp)->records_unread); + v_set(config, &shmp(handle, pages->shmp)->records_unread, + v_read(config, &shmp(handle, pages->shmp)->records_commit)); + v_set(config, &shmp(handle, pages->shmp)->records_commit, 0); return overruns; } static inline -void subbuffer_set_data_size(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, +void subbuffer_set_data_size(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, unsigned long idx, - unsigned long data_size) + unsigned long data_size, + struct lttng_ust_shm_handle *handle) { - struct lib_ring_buffer_backend_pages *pages; + struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages; unsigned long sb_bindex; - sb_bindex = subbuffer_id_get_index(config, shmp(bufb->buf_wsb)[idx].id); - pages = shmp(bufb->array)[sb_bindex]; - pages->data_size = data_size; + sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id); + pages = shmp_index(handle, bufb->array, sb_bindex); + shmp(handle, pages->shmp)->data_size = data_size; } static inline unsigned long subbuffer_get_read_data_size( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb) + const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + struct lttng_ust_shm_handle *handle) { - struct lib_ring_buffer_backend_pages *pages; + struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages; unsigned long sb_bindex; sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id); - pages = shmp(bufb->array)[sb_bindex]; - return pages->data_size; + pages = shmp_index(handle, bufb->array, sb_bindex); + return shmp(handle, pages->shmp)->data_size; } static inline unsigned long subbuffer_get_data_size( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) + const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + unsigned long idx, + struct lttng_ust_shm_handle *handle) { - struct lib_ring_buffer_backend_pages *pages; + struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages; unsigned long sb_bindex; - sb_bindex = subbuffer_id_get_index(config, shmp(bufb->buf_wsb)[idx].id); - pages = shmp(bufb->array)[sb_bindex]; - return pages->data_size; + sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id); + pages = shmp_index(handle, bufb->array, sb_bindex); + return shmp(handle, pages->shmp)->data_size; } /** @@ -292,9 +301,10 @@ unsigned long subbuffer_get_data_size( * writer. */ static inline -void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) +void lib_ring_buffer_clear_noref(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + unsigned long idx, + struct lttng_ust_shm_handle *handle) { unsigned long id, new_id; @@ -305,10 +315,10 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, * Performing a volatile access to read the sb_pages, because we want to * read a coherent version of the pointer and the associated noref flag. */ - id = CMM_ACCESS_ONCE(shmp(bufb->buf_wsb)[idx].id); + id = CMM_ACCESS_ONCE(shmp_index(handle, bufb->buf_wsb, idx)->id); for (;;) { /* This check is called on the fast path for each record. */ - if (likely(!subbuffer_id_is_noref(config, id))) { + if (caa_likely(!subbuffer_id_is_noref(config, id))) { /* * Store after load dependency ordering the writes to * the subbuffer after load and test of the noref flag @@ -319,8 +329,8 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, } new_id = id; subbuffer_id_clear_noref(config, &new_id); - new_id = uatomic_cmpxchg(&shmp(bufb->buf_wsb)[idx].id, id, new_id); - if (likely(new_id == id)) + new_id = uatomic_cmpxchg(&shmp_index(handle, bufb->buf_wsb, idx)->id, id, new_id); + if (caa_likely(new_id == id)) break; id = new_id; } @@ -331,9 +341,10 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, * called by writer. */ static inline -void lib_ring_buffer_set_noref_offset(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx, unsigned long offset) +void lib_ring_buffer_set_noref_offset(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, + unsigned long idx, unsigned long offset, + struct lttng_ust_shm_handle *handle) { if (config->mode != RING_BUFFER_OVERWRITE) return; @@ -349,25 +360,26 @@ void lib_ring_buffer_set_noref_offset(const struct lib_ring_buffer_config *confi * subbuffer_set_noref() uses a volatile store to deal with concurrent * readers of the noref flag. */ - CHAN_WARN_ON(bufb->chan, - subbuffer_id_is_noref(config, shmp(bufb->buf_wsb)[idx].id)); + CHAN_WARN_ON(shmp(handle, bufb->chan), + subbuffer_id_is_noref(config, shmp_index(handle, bufb->buf_wsb, idx)->id)); /* * Memory barrier that ensures counter stores are ordered before set * noref and offset. */ cmm_smp_mb(); - subbuffer_id_set_noref_offset(config, &shmp(bufb->buf_wsb)[idx].id, offset); + subbuffer_id_set_noref_offset(config, &shmp_index(handle, bufb->buf_wsb, idx)->id, offset); } /** * update_read_sb_index - Read-side subbuffer index update. */ static inline -int update_read_sb_index(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, +int update_read_sb_index(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_backend *bufb, struct channel_backend *chanb, unsigned long consumed_idx, - unsigned long consumed_count) + unsigned long consumed_count, + struct lttng_ust_shm_handle *handle) { unsigned long old_id, new_id; @@ -378,32 +390,36 @@ int update_read_sb_index(const struct lib_ring_buffer_config *config, * old_wpage, because the value read will be confirmed by the * following cmpxchg(). */ - old_id = shmp(bufb->buf_wsb)[consumed_idx].id; - if (unlikely(!subbuffer_id_is_noref(config, old_id))) + old_id = shmp_index(handle, bufb->buf_wsb, consumed_idx)->id; + if (caa_unlikely(!subbuffer_id_is_noref(config, old_id))) return -EAGAIN; /* * Make sure the offset count we are expecting matches the one * indicated by the writer. */ - if (unlikely(!subbuffer_id_compare_offset(config, old_id, + if (caa_unlikely(!subbuffer_id_compare_offset(config, old_id, consumed_count))) return -EAGAIN; - CHAN_WARN_ON(bufb->chan, + CHAN_WARN_ON(shmp(handle, bufb->chan), !subbuffer_id_is_noref(config, bufb->buf_rsb.id)); subbuffer_id_set_noref_offset(config, &bufb->buf_rsb.id, consumed_count); - new_id = uatomic_cmpxchg(&shmp(bufb->buf_wsb)[consumed_idx].id, old_id, + new_id = uatomic_cmpxchg(&shmp_index(handle, bufb->buf_wsb, consumed_idx)->id, old_id, bufb->buf_rsb.id); - if (unlikely(old_id != new_id)) + if (caa_unlikely(old_id != new_id)) return -EAGAIN; bufb->buf_rsb.id = new_id; } else { /* No page exchange, use the writer page directly */ - bufb->buf_rsb.id = shmp(bufb->buf_wsb)[consumed_idx].id; + bufb->buf_rsb.id = shmp_index(handle, bufb->buf_wsb, consumed_idx)->id; } return 0; } +#ifndef inline_memcpy +#define inline_memcpy(dest, src, n) memcpy(dest, src, n) +#endif + /* * Use the architecture-specific memcpy implementation for constant-sized * inputs, but rely on an inline memcpy for length statically unknown. @@ -418,4 +434,61 @@ do { \ inline_memcpy(dest, src, __len); \ } while (0) +/* arch-agnostic implementation */ + +static inline int fls(unsigned int x) +{ + int r = 32; + + if (!x) + return 0; + if (!(x & 0xFFFF0000U)) { + x <<= 16; + r -= 16; + } + if (!(x & 0xFF000000U)) { + x <<= 8; + r -= 8; + } + if (!(x & 0xF0000000U)) { + x <<= 4; + r -= 4; + } + if (!(x & 0xC0000000U)) { + x <<= 2; + r -= 2; + } + if (!(x & 0x80000000U)) { + x <<= 1; + r -= 1; + } + return r; +} + +static inline int get_count_order(unsigned int count) +{ + int order; + + order = fls(count) - 1; + if (count & (count - 1)) + order++; + return order; +} + +static inline +unsigned int hweight32(unsigned int value) +{ + unsigned int r; + + r = value; + r = r - ((r >> 1) & 0x55555555); + r = (r & 0x33333333) + ((r >> 2) & 0x33333333); + r += r >> 4; + r &= 0x0F0F0F0F; + r += r >> 8; + r += r >> 16; + r &= 0x000000FF; + return r; +} + #endif /* _LINUX_RING_BUFFER_BACKEND_INTERNAL_H */