X-Git-Url: https://git.lttng.org/?a=blobdiff_plain;f=src%2Flib%2Fringbuffer%2Fring_buffer_backend.c;h=9a339be0a6e9b7040abce39b9585a2f2e5d95f48;hb=ffcc873470121ef1ebb110df3d9038a38d9cb7cb;hp=76f9b4252a3a6bd58d88ee09d42b442627e4f69e;hpb=0d1080f40a7ac09fce7120398746b1bfab3d32d8;p=lttng-modules.git diff --git a/src/lib/ringbuffer/ring_buffer_backend.c b/src/lib/ringbuffer/ring_buffer_backend.c index 76f9b425..9a339be0 100644 --- a/src/lib/ringbuffer/ring_buffer_backend.c +++ b/src/lib/ringbuffer/ring_buffer_backend.c @@ -12,10 +12,10 @@ #include #include #include -#include #include #include +#include #include #include /* for wrapper_vmalloc_sync_mappings() */ #include @@ -445,14 +445,14 @@ int channel_backend_init(struct channel_backend *chanb, chanb->cpu_hp_notifier.priority = 5; register_hotcpu_notifier(&chanb->cpu_hp_notifier); - get_online_cpus(); + lttng_cpus_read_lock(); for_each_online_cpu(i) { ret = lib_ring_buffer_create(per_cpu_ptr(chanb->buf, i), chanb, i); if (ret) goto free_bufs; /* cpu hotplug locked */ } - put_online_cpus(); + lttng_cpus_read_unlock(); #else for_each_possible_cpu(i) { ret = lib_ring_buffer_create(per_cpu_ptr(chanb->buf, i), @@ -485,7 +485,7 @@ free_bufs: */ #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */ #ifdef CONFIG_HOTPLUG_CPU - put_online_cpus(); + lttng_cpus_read_unlock(); unregister_hotcpu_notifier(&chanb->cpu_hp_notifier); #endif #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */ @@ -565,21 +565,17 @@ void channel_backend_free(struct channel_backend *chanb) * @offset : offset within the buffer * @src : source address * @len : length to write - * @pagecpy : page size copied so far */ void _lib_ring_buffer_write(struct lttng_kernel_ring_buffer_backend *bufb, size_t offset, - const void *src, size_t len, size_t pagecpy) + const void *src, size_t len) { struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; - size_t sbidx, index; + size_t sbidx, index, bytes_left_in_page; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; do { - len -= pagecpy; - src += pagecpy; - offset += pagecpy; sbidx = offset >> chanb->subbuf_size_order; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; @@ -589,7 +585,7 @@ void _lib_ring_buffer_write(struct lttng_kernel_ring_buffer_backend *bufb, size_ */ CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_wsb[sbidx].id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; @@ -598,33 +594,31 @@ void _lib_ring_buffer_write(struct lttng_kernel_ring_buffer_backend *bufb, size_ lib_ring_buffer_do_copy(config, rpages->p[index].virt + (offset & ~PAGE_MASK), - src, pagecpy); - } while (unlikely(len != pagecpy)); + src, bytes_left_in_page); + len -= bytes_left_in_page; + src += bytes_left_in_page; + offset += bytes_left_in_page; + } while (unlikely(len)); } EXPORT_SYMBOL_GPL(_lib_ring_buffer_write); - /** * lib_ring_buffer_memset - write len bytes of c to a ring_buffer buffer. * @bufb : buffer backend * @offset : offset within the buffer * @c : the byte to write * @len : length to write - * @pagecpy : page size copied so far */ void _lib_ring_buffer_memset(struct lttng_kernel_ring_buffer_backend *bufb, - size_t offset, - int c, size_t len, size_t pagecpy) + size_t offset, int c, size_t len) { struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; - size_t sbidx, index; + size_t sbidx, index, bytes_left_in_page; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; do { - len -= pagecpy; - offset += pagecpy; sbidx = offset >> chanb->subbuf_size_order; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; @@ -634,7 +628,7 @@ void _lib_ring_buffer_memset(struct lttng_kernel_ring_buffer_backend *bufb, */ CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_wsb[sbidx].id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; @@ -642,8 +636,10 @@ void _lib_ring_buffer_memset(struct lttng_kernel_ring_buffer_backend *bufb, && subbuffer_id_is_noref(config, id)); lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), - c, pagecpy); - } while (unlikely(len != pagecpy)); + c, bytes_left_in_page); + len -= bytes_left_in_page; + offset += bytes_left_in_page; + } while (unlikely(len)); } EXPORT_SYMBOL_GPL(_lib_ring_buffer_memset); @@ -653,26 +649,20 @@ EXPORT_SYMBOL_GPL(_lib_ring_buffer_memset); * @offset : offset within the buffer * @src : source address * @len : length to write - * @pagecpy : page size copied so far * @pad : character to use for padding */ void _lib_ring_buffer_strcpy(struct lttng_kernel_ring_buffer_backend *bufb, - size_t offset, const char *src, size_t len, - size_t pagecpy, int pad) + size_t offset, const char *src, size_t len, int pad) { struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; - size_t sbidx, index; + size_t sbidx, index, bytes_left_in_page; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; - int src_terminated = 0; + bool src_terminated = false; CHAN_WARN_ON(chanb, !len); - offset += pagecpy; do { - len -= pagecpy; - if (!src_terminated) - src += pagecpy; sbidx = offset >> chanb->subbuf_size_order; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; @@ -682,7 +672,7 @@ void _lib_ring_buffer_strcpy(struct lttng_kernel_ring_buffer_backend *bufb, */ CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_wsb[sbidx].id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; @@ -692,8 +682,8 @@ void _lib_ring_buffer_strcpy(struct lttng_kernel_ring_buffer_backend *bufb, if (likely(!src_terminated)) { size_t count, to_copy; - to_copy = pagecpy; - if (pagecpy == len) + to_copy = bytes_left_in_page; + if (bytes_left_in_page == len) to_copy--; /* Final '\0' */ count = lib_ring_buffer_do_strcpy(config, rpages->p[index].virt @@ -705,7 +695,7 @@ void _lib_ring_buffer_strcpy(struct lttng_kernel_ring_buffer_backend *bufb, size_t pad_len = to_copy - count; /* Next pages will have padding */ - src_terminated = 1; + src_terminated = true; lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), pad, pad_len); @@ -714,15 +704,19 @@ void _lib_ring_buffer_strcpy(struct lttng_kernel_ring_buffer_backend *bufb, } else { size_t pad_len; - pad_len = pagecpy; - if (pagecpy == len) + pad_len = bytes_left_in_page; + if (bytes_left_in_page == len) pad_len--; /* Final '\0' */ lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), pad, pad_len); offset += pad_len; } - } while (unlikely(len != pagecpy)); + len -= bytes_left_in_page; + if (!src_terminated) + src += bytes_left_in_page; + } while (unlikely(len)); + /* Ending '\0' */ lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), '\0', 1); @@ -814,28 +808,22 @@ EXPORT_SYMBOL_GPL(_lib_ring_buffer_pstrcpy); * @offset : offset within the buffer * @src : source address * @len : length to write - * @pagecpy : page size copied so far * * This function deals with userspace pointers, it should never be called * directly without having the src pointer checked with access_ok() * previously. */ void _lib_ring_buffer_copy_from_user_inatomic(struct lttng_kernel_ring_buffer_backend *bufb, - size_t offset, - const void __user *src, size_t len, - size_t pagecpy) + size_t offset, const void __user *src, size_t len) { struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; - size_t sbidx, index; + size_t sbidx, index, bytes_left_in_page; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; int ret; do { - len -= pagecpy; - src += pagecpy; - offset += pagecpy; sbidx = offset >> chanb->subbuf_size_order; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; @@ -845,7 +833,7 @@ void _lib_ring_buffer_copy_from_user_inatomic(struct lttng_kernel_ring_buffer_ba */ CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_wsb[sbidx].id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; @@ -853,13 +841,16 @@ void _lib_ring_buffer_copy_from_user_inatomic(struct lttng_kernel_ring_buffer_ba && subbuffer_id_is_noref(config, id)); ret = lib_ring_buffer_do_copy_from_user_inatomic(rpages->p[index].virt + (offset & ~PAGE_MASK), - src, pagecpy) != 0; + src, bytes_left_in_page) != 0; if (ret > 0) { /* Copy failed. */ - _lib_ring_buffer_memset(bufb, offset, 0, len, 0); + _lib_ring_buffer_memset(bufb, offset, 0, len); break; /* stop copy */ } - } while (unlikely(len != pagecpy)); + len -= bytes_left_in_page; + src += bytes_left_in_page; + offset += bytes_left_in_page; + } while (unlikely(len)); } EXPORT_SYMBOL_GPL(_lib_ring_buffer_copy_from_user_inatomic); @@ -869,7 +860,6 @@ EXPORT_SYMBOL_GPL(_lib_ring_buffer_copy_from_user_inatomic); * @offset : offset within the buffer * @src : source address * @len : length to write - * @pagecpy : page size copied so far * @pad : character to use for padding * * This function deals with userspace pointers, it should never be called @@ -877,21 +867,16 @@ EXPORT_SYMBOL_GPL(_lib_ring_buffer_copy_from_user_inatomic); * previously. */ void _lib_ring_buffer_strcpy_from_user_inatomic(struct lttng_kernel_ring_buffer_backend *bufb, - size_t offset, const char __user *src, size_t len, - size_t pagecpy, int pad) + size_t offset, const char __user *src, size_t len, int pad) { struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; - size_t sbidx, index; + size_t sbidx, index, bytes_left_in_page; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; - int src_terminated = 0; + bool src_terminated = false; - offset += pagecpy; do { - len -= pagecpy; - if (!src_terminated) - src += pagecpy; sbidx = offset >> chanb->subbuf_size_order; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; @@ -901,7 +886,7 @@ void _lib_ring_buffer_strcpy_from_user_inatomic(struct lttng_kernel_ring_buffer_ */ CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_wsb[sbidx].id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; @@ -911,8 +896,8 @@ void _lib_ring_buffer_strcpy_from_user_inatomic(struct lttng_kernel_ring_buffer_ if (likely(!src_terminated)) { size_t count, to_copy; - to_copy = pagecpy; - if (pagecpy == len) + to_copy = bytes_left_in_page; + if (bytes_left_in_page == len) to_copy--; /* Final '\0' */ count = lib_ring_buffer_do_strcpy_from_user_inatomic(config, rpages->p[index].virt @@ -924,7 +909,7 @@ void _lib_ring_buffer_strcpy_from_user_inatomic(struct lttng_kernel_ring_buffer_ size_t pad_len = to_copy - count; /* Next pages will have padding */ - src_terminated = 1; + src_terminated = true; lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), pad, pad_len); @@ -933,15 +918,19 @@ void _lib_ring_buffer_strcpy_from_user_inatomic(struct lttng_kernel_ring_buffer_ } else { size_t pad_len; - pad_len = pagecpy; - if (pagecpy == len) + pad_len = bytes_left_in_page; + if (bytes_left_in_page == len) pad_len--; /* Final '\0' */ lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), pad, pad_len); offset += pad_len; } - } while (unlikely(len != pagecpy)); + len -= bytes_left_in_page; + if (!src_terminated) + src += bytes_left_in_page; + } while (unlikely(len)); + /* Ending '\0' */ lib_ring_buffer_do_memset(rpages->p[index].virt + (offset & ~PAGE_MASK), '\0', 1); @@ -1046,7 +1035,7 @@ size_t lib_ring_buffer_read(struct lttng_kernel_ring_buffer_backend *bufb, size_ { struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; - size_t index, pagecpy, orig_len; + size_t index, bytes_left_in_page, orig_len; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; @@ -1056,19 +1045,19 @@ size_t lib_ring_buffer_read(struct lttng_kernel_ring_buffer_backend *bufb, size_ if (unlikely(!len)) return 0; for (;;) { - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_rsb.id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; CHAN_WARN_ON(chanb, config->mode == RING_BUFFER_OVERWRITE && subbuffer_id_is_noref(config, id)); memcpy(dest, rpages->p[index].virt + (offset & ~PAGE_MASK), - pagecpy); - len -= pagecpy; + bytes_left_in_page); + len -= bytes_left_in_page; if (likely(!len)) break; - dest += pagecpy; - offset += pagecpy; + dest += bytes_left_in_page; + offset += bytes_left_in_page; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; /* * Underlying layer should never ask for reads across @@ -1098,7 +1087,7 @@ int __lib_ring_buffer_copy_to_user(struct lttng_kernel_ring_buffer_backend *bufb struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; size_t index; - ssize_t pagecpy; + ssize_t bytes_left_in_page; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; @@ -1107,7 +1096,7 @@ int __lib_ring_buffer_copy_to_user(struct lttng_kernel_ring_buffer_backend *bufb if (unlikely(!len)) return 0; for (;;) { - pagecpy = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); + bytes_left_in_page = min_t(size_t, len, PAGE_SIZE - (offset & ~PAGE_MASK)); id = bufb->buf_rsb.id; sb_bindex = subbuffer_id_get_index(config, id); rpages = bufb->array[sb_bindex]; @@ -1115,13 +1104,13 @@ int __lib_ring_buffer_copy_to_user(struct lttng_kernel_ring_buffer_backend *bufb && subbuffer_id_is_noref(config, id)); if (__copy_to_user(dest, rpages->p[index].virt + (offset & ~PAGE_MASK), - pagecpy)) + bytes_left_in_page)) return -EFAULT; - len -= pagecpy; + len -= bytes_left_in_page; if (likely(!len)) break; - dest += pagecpy; - offset += pagecpy; + dest += bytes_left_in_page; + offset += bytes_left_in_page; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; /* * Underlying layer should never ask for reads across @@ -1150,7 +1139,7 @@ int lib_ring_buffer_read_cstr(struct lttng_kernel_ring_buffer_backend *bufb, siz struct channel_backend *chanb = &bufb->chan->backend; const struct lttng_kernel_ring_buffer_config *config = &chanb->config; size_t index; - ssize_t pagecpy, pagelen, strpagelen, orig_offset; + ssize_t bytes_left_in_page, pagelen, strpagelen, orig_offset; char *str; struct lttng_kernel_ring_buffer_backend_pages *rpages; unsigned long sb_bindex, id; @@ -1170,12 +1159,12 @@ int lib_ring_buffer_read_cstr(struct lttng_kernel_ring_buffer_backend *bufb, siz pagelen = PAGE_SIZE - (offset & ~PAGE_MASK); strpagelen = strnlen(str, pagelen); if (len) { - pagecpy = min_t(size_t, len, strpagelen); + bytes_left_in_page = min_t(size_t, len, strpagelen); if (dest) { - memcpy(dest, str, pagecpy); - dest += pagecpy; + memcpy(dest, str, bytes_left_in_page); + dest += bytes_left_in_page; } - len -= pagecpy; + len -= bytes_left_in_page; } offset += strpagelen; index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;