X-Git-Url: http://git.lttng.org/?a=blobdiff_plain;f=lib%2Fringbuffer%2Fring_buffer_backend.c;h=8207f764377a34f5cb0fda0e41da8e56985dbd45;hb=1f0ab1eb0409d23de5f67cc588c3ea4cee4d10e0;hp=c7f2fe9d7b6226b33738d2fbabb379b42999ef93;hpb=769b62a02f62ff8db93f109d9f63a1df6ea9990b;p=lttng-modules.git diff --git a/lib/ringbuffer/ring_buffer_backend.c b/lib/ringbuffer/ring_buffer_backend.c index c7f2fe9d..8207f764 100644 --- a/lib/ringbuffer/ring_buffer_backend.c +++ b/lib/ringbuffer/ring_buffer_backend.c @@ -1,21 +1,8 @@ -/* +/* SPDX-License-Identifier: (GPL-2.0 OR LGPL-2.1) + * * ring_buffer_backend.c * * Copyright (C) 2005-2012 Mathieu Desnoyers - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; only - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include @@ -25,8 +12,10 @@ #include #include #include +#include #include #include +#include #include /* for wrapper_vmalloc_sync_all() */ #include @@ -55,6 +44,23 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config unsigned long i; num_pages = size >> PAGE_SHIFT; + + /* + * Verify that the number of pages requested for that buffer is smaller + * than the number of available pages on the system. si_mem_available() + * returns an _estimate_ of the number of available pages. + */ + if (num_pages > si_mem_available()) + goto not_enough_pages; + + /* + * Set the current user thread as the first target of the OOM killer. + * If the estimate received by si_mem_available() was off, and we do + * end up running out of memory because of this buffer allocation, we + * want to kill the offending app first. + */ + set_current_oom_origin(); + num_pages_per_subbuf = num_pages >> get_count_order(num_subbuf); subbuf_size = chanb->subbuf_size; num_subbuf_alloc = num_subbuf; @@ -64,22 +70,23 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config num_subbuf_alloc++; } - pages = kmalloc_node(ALIGN(sizeof(*pages) * num_pages, + pages = vmalloc_node(ALIGN(sizeof(*pages) * num_pages, 1 << INTERNODE_CACHE_SHIFT), - GFP_KERNEL, cpu_to_node(max(bufb->cpu, 0))); + cpu_to_node(max(bufb->cpu, 0))); if (unlikely(!pages)) goto pages_error; - bufb->array = kmalloc_node(ALIGN(sizeof(*bufb->array) + bufb->array = lttng_kvmalloc_node(ALIGN(sizeof(*bufb->array) * num_subbuf_alloc, 1 << INTERNODE_CACHE_SHIFT), - GFP_KERNEL, cpu_to_node(max(bufb->cpu, 0))); + GFP_KERNEL | __GFP_NOWARN, + cpu_to_node(max(bufb->cpu, 0))); if (unlikely(!bufb->array)) goto array_error; for (i = 0; i < num_pages; i++) { pages[i] = alloc_pages_node(cpu_to_node(max(bufb->cpu, 0)), - GFP_KERNEL | __GFP_ZERO, 0); + GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0); if (unlikely(!pages[i])) goto depopulate; } @@ -88,22 +95,24 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config /* Allocate backend pages array elements */ for (i = 0; i < num_subbuf_alloc; i++) { bufb->array[i] = - kzalloc_node(ALIGN( + lttng_kvzalloc_node(ALIGN( sizeof(struct lib_ring_buffer_backend_pages) + sizeof(struct lib_ring_buffer_backend_page) * num_pages_per_subbuf, 1 << INTERNODE_CACHE_SHIFT), - GFP_KERNEL, cpu_to_node(max(bufb->cpu, 0))); + GFP_KERNEL | __GFP_NOWARN, + cpu_to_node(max(bufb->cpu, 0))); if (!bufb->array[i]) goto free_array; } /* Allocate write-side subbuffer table */ - bufb->buf_wsb = kzalloc_node(ALIGN( + bufb->buf_wsb = lttng_kvzalloc_node(ALIGN( sizeof(struct lib_ring_buffer_backend_subbuffer) * num_subbuf, 1 << INTERNODE_CACHE_SHIFT), - GFP_KERNEL, cpu_to_node(max(bufb->cpu, 0))); + GFP_KERNEL | __GFP_NOWARN, + cpu_to_node(max(bufb->cpu, 0))); if (unlikely(!bufb->buf_wsb)) goto free_array; @@ -118,11 +127,12 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config bufb->buf_rsb.id = subbuffer_id(config, 0, 1, 0); /* Allocate subbuffer packet counter table */ - bufb->buf_cnt = kzalloc_node(ALIGN( + bufb->buf_cnt = lttng_kvzalloc_node(ALIGN( sizeof(struct lib_ring_buffer_backend_counts) * num_subbuf, 1 << INTERNODE_CACHE_SHIFT), - GFP_KERNEL, cpu_to_node(max(bufb->cpu, 0))); + GFP_KERNEL | __GFP_NOWARN, + cpu_to_node(max(bufb->cpu, 0))); if (unlikely(!bufb->buf_cnt)) goto free_wsb; @@ -145,22 +155,25 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config * will not fault. */ wrapper_vmalloc_sync_all(); - kfree(pages); + clear_current_oom_origin(); + vfree(pages); return 0; free_wsb: - kfree(bufb->buf_wsb); + lttng_kvfree(bufb->buf_wsb); free_array: for (i = 0; (i < num_subbuf_alloc && bufb->array[i]); i++) - kfree(bufb->array[i]); + lttng_kvfree(bufb->array[i]); depopulate: /* Free all allocated pages */ for (i = 0; (i < num_pages && pages[i]); i++) __free_page(pages[i]); - kfree(bufb->array); + lttng_kvfree(bufb->array); array_error: - kfree(pages); + vfree(pages); pages_error: + clear_current_oom_origin(); +not_enough_pages: return -ENOMEM; } @@ -186,14 +199,14 @@ void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend *bufb) if (chanb->extra_reader_sb) num_subbuf_alloc++; - kfree(bufb->buf_wsb); - kfree(bufb->buf_cnt); + lttng_kvfree(bufb->buf_wsb); + lttng_kvfree(bufb->buf_cnt); for (i = 0; i < num_subbuf_alloc; i++) { for (j = 0; j < bufb->num_pages_per_subbuf; j++) __free_page(pfn_to_page(bufb->array[i]->p[j].pfn)); - kfree(bufb->array[i]); + lttng_kvfree(bufb->array[i]); } - kfree(bufb->array); + lttng_kvfree(bufb->array); bufb->allocated = 0; } @@ -244,7 +257,42 @@ void channel_backend_reset(struct channel_backend *chanb) chanb->start_tsc = config->cb.ring_buffer_clock_read(chan); } +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) + +/* + * No need to implement a "dead" callback to do a buffer switch here, + * because it will happen when tracing is stopped, or will be done by + * switch timer CPU DEAD callback. + * We don't free buffers when CPU go away, because it would make trace + * data vanish, which is unwanted. + */ +int lttng_cpuhp_rb_backend_prepare(unsigned int cpu, + struct lttng_cpuhp_node *node) +{ + struct channel_backend *chanb = container_of(node, + struct channel_backend, cpuhp_prepare); + const struct lib_ring_buffer_config *config = &chanb->config; + struct lib_ring_buffer *buf; + int ret; + + CHAN_WARN_ON(chanb, config->alloc == RING_BUFFER_ALLOC_GLOBAL); + + buf = per_cpu_ptr(chanb->buf, cpu); + ret = lib_ring_buffer_create(buf, chanb, cpu); + if (ret) { + printk(KERN_ERR + "ring_buffer_cpu_hp_callback: cpu %d " + "buffer creation failed\n", cpu); + return ret; + } + return 0; +} +EXPORT_SYMBOL_GPL(lttng_cpuhp_rb_backend_prepare); + +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ + #ifdef CONFIG_HOTPLUG_CPU + /** * lib_ring_buffer_cpu_hp_callback - CPU hotplug callback * @nb: notifier block @@ -288,8 +336,11 @@ int lib_ring_buffer_cpu_hp_callback(struct notifier_block *nb, } return NOTIFY_OK; } + #endif +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ + /** * channel_backend_init - initialize a channel backend * @chanb: channel backend @@ -366,39 +417,50 @@ int channel_backend_init(struct channel_backend *chanb, if (!chanb->buf) goto free_cpumask; - /* - * In case of non-hotplug cpu, if the ring-buffer is allocated - * in early initcall, it will not be notified of secondary cpus. - * In that off case, we need to allocate for all possible cpus. - */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) + chanb->cpuhp_prepare.component = LTTNG_RING_BUFFER_BACKEND; + ret = cpuhp_state_add_instance(lttng_rb_hp_prepare, + &chanb->cpuhp_prepare.node); + if (ret) + goto free_bufs; +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ + + { + /* + * In case of non-hotplug cpu, if the ring-buffer is allocated + * in early initcall, it will not be notified of secondary cpus. + * In that off case, we need to allocate for all possible cpus. + */ #ifdef CONFIG_HOTPLUG_CPU - /* - * buf->backend.allocated test takes care of concurrent CPU - * hotplug. - * Priority higher than frontend, so we create the ring buffer - * before we start the timer. - */ - chanb->cpu_hp_notifier.notifier_call = - lib_ring_buffer_cpu_hp_callback; - chanb->cpu_hp_notifier.priority = 5; - register_hotcpu_notifier(&chanb->cpu_hp_notifier); - - get_online_cpus(); - for_each_online_cpu(i) { - ret = lib_ring_buffer_create(per_cpu_ptr(chanb->buf, i), - chanb, i); - if (ret) - goto free_bufs; /* cpu hotplug locked */ - } - put_online_cpus(); + /* + * buf->backend.allocated test takes care of concurrent CPU + * hotplug. + * Priority higher than frontend, so we create the ring buffer + * before we start the timer. + */ + chanb->cpu_hp_notifier.notifier_call = + lib_ring_buffer_cpu_hp_callback; + chanb->cpu_hp_notifier.priority = 5; + register_hotcpu_notifier(&chanb->cpu_hp_notifier); + + get_online_cpus(); + for_each_online_cpu(i) { + ret = lib_ring_buffer_create(per_cpu_ptr(chanb->buf, i), + chanb, i); + if (ret) + goto free_bufs; /* cpu hotplug locked */ + } + put_online_cpus(); #else - for_each_possible_cpu(i) { - ret = lib_ring_buffer_create(per_cpu_ptr(chanb->buf, i), - chanb, i); - if (ret) - goto free_bufs; /* cpu hotplug locked */ - } + for_each_possible_cpu(i) { + ret = lib_ring_buffer_create(per_cpu_ptr(chanb->buf, i), + chanb, i); + if (ret) + goto free_bufs; + } #endif + } +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ } else { chanb->buf = kzalloc(sizeof(struct lib_ring_buffer), GFP_KERNEL); if (!chanb->buf) @@ -413,17 +475,26 @@ int channel_backend_init(struct channel_backend *chanb, free_bufs: if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) + /* + * Teardown of lttng_rb_hp_prepare instance + * on "add" error is handled within cpu hotplug, + * no teardown to do from the caller. + */ +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ +#ifdef CONFIG_HOTPLUG_CPU + put_online_cpus(); + unregister_hotcpu_notifier(&chanb->cpu_hp_notifier); +#endif +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ for_each_possible_cpu(i) { - struct lib_ring_buffer *buf = per_cpu_ptr(chanb->buf, i); + struct lib_ring_buffer *buf = + per_cpu_ptr(chanb->buf, i); if (!buf->backend.allocated) continue; lib_ring_buffer_free(buf); } -#ifdef CONFIG_HOTPLUG_CPU - put_online_cpus(); - unregister_hotcpu_notifier(&chanb->cpu_hp_notifier); -#endif free_percpu(chanb->buf); } else kfree(chanb->buf); @@ -443,8 +514,17 @@ void channel_backend_unregister_notifiers(struct channel_backend *chanb) { const struct lib_ring_buffer_config *config = &chanb->config; - if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) + if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) + int ret; + + ret = cpuhp_state_remove_instance(lttng_rb_hp_prepare, + &chanb->cpuhp_prepare.node); + WARN_ON(ret); +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ unregister_hotcpu_notifier(&chanb->cpu_hp_notifier); +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ + } } /**