From c50bdba26664558b966a324cd382bd2959f0444e Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Mon, 13 Apr 2020 11:09:13 -0400 Subject: [PATCH] wrapper: remove vmalloc_sync_all --- lib/prio_heap/lttng_prio_heap.c | 9 +- lib/ringbuffer/ring_buffer_backend.c | 33 ++-- lib/ringbuffer/ring_buffer_frontend.c | 20 ++- lttng-abi.c | 2 - lttng-context-callstack.c | 2 - lttng-context-cgroup-ns.c | 2 - lttng-context-cpu-id.c | 2 - lttng-context-egid.c | 2 - lttng-context-euid.c | 2 - lttng-context-gid.c | 2 - lttng-context-hostname.c | 2 - lttng-context-interruptible.c | 2 - lttng-context-ipc-ns.c | 2 - lttng-context-migratable.c | 2 - lttng-context-mnt-ns.c | 2 - lttng-context-need-reschedule.c | 2 - lttng-context-net-ns.c | 2 - lttng-context-nice.c | 2 - lttng-context-perf-counters.c | 9 +- lttng-context-pid-ns.c | 2 - lttng-context-pid.c | 2 - lttng-context-ppid.c | 2 - lttng-context-preemptible.c | 2 - lttng-context-prio.c | 2 - lttng-context-procname.c | 2 - lttng-context-sgid.c | 2 - lttng-context-suid.c | 2 - lttng-context-tid.c | 2 - lttng-context-uid.c | 2 - lttng-context-user-ns.c | 2 - lttng-context-uts-ns.c | 2 - lttng-context-vegid.c | 2 - lttng-context-veuid.c | 2 - lttng-context-vgid.c | 2 - lttng-context-vpid.c | 2 - lttng-context-vppid.c | 2 - lttng-context-vsgid.c | 2 - lttng-context-vsuid.c | 2 - lttng-context-vtid.c | 2 - lttng-context-vuid.c | 2 - lttng-context.c | 8 +- lttng-events.c | 24 +-- lttng-ring-buffer-client.h | 6 - lttng-ring-buffer-metadata-client.h | 6 - lttng-syscalls.c | 2 - lttng-tracer.h | 1 - probes/lttng-kprobes.c | 8 - probes/lttng-kretprobes.c | 8 - probes/lttng-tracepoint-event-impl.h | 2 - probes/lttng-uprobes.c | 4 - probes/lttng.c | 3 - tests/probes/lttng-test.c | 1 - wrapper/vmalloc.h | 210 -------------------------- 53 files changed, 41 insertions(+), 385 deletions(-) delete mode 100644 wrapper/vmalloc.h diff --git a/lib/prio_heap/lttng_prio_heap.c b/lib/prio_heap/lttng_prio_heap.c index 04438bbf..4c62782f 100644 --- a/lib/prio_heap/lttng_prio_heap.c +++ b/lib/prio_heap/lttng_prio_heap.c @@ -9,7 +9,7 @@ #include #include -#include +#include #ifdef DEBUG_HEAP void lttng_check_heap(const struct lttng_ptr_heap *heap) @@ -54,12 +54,13 @@ int heap_grow(struct lttng_ptr_heap *heap, size_t new_len) return 0; heap->alloc_len = max_t(size_t, new_len, heap->alloc_len << 1); - new_ptrs = lttng_kvmalloc(heap->alloc_len * sizeof(void *), heap->gfpmask); + new_ptrs = kvmalloc_node(heap->alloc_len * sizeof(void *), heap->gfpmask, + NUMA_NO_NODE); if (!new_ptrs) return -ENOMEM; if (heap->ptrs) memcpy(new_ptrs, heap->ptrs, heap->len * sizeof(void *)); - lttng_kvfree(heap->ptrs); + kvfree(heap->ptrs); heap->ptrs = new_ptrs; return 0; } @@ -93,7 +94,7 @@ int lttng_heap_init(struct lttng_ptr_heap *heap, size_t alloc_len, void lttng_heap_free(struct lttng_ptr_heap *heap) { - lttng_kvfree(heap->ptrs); + kvfree(heap->ptrs); } static void heapify(struct lttng_ptr_heap *heap, size_t i) diff --git a/lib/ringbuffer/ring_buffer_backend.c b/lib/ringbuffer/ring_buffer_backend.c index 468b0e9b..435129cf 100644 --- a/lib/ringbuffer/ring_buffer_backend.c +++ b/lib/ringbuffer/ring_buffer_backend.c @@ -17,7 +17,6 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ #include #include #include @@ -78,14 +77,13 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config if (unlikely(!pages)) goto pages_error; - bufb->array = lttng_kvmalloc_node(ALIGN(sizeof(*bufb->array) - * num_subbuf_alloc, - 1 << INTERNODE_CACHE_SHIFT), + bufb->array = kvmalloc_node(ALIGN(sizeof(*bufb->array) + * num_subbuf_alloc, + 1 << INTERNODE_CACHE_SHIFT), GFP_KERNEL | __GFP_NOWARN, cpu_to_node(max(bufb->cpu, 0))); if (unlikely(!bufb->array)) goto array_error; - for (i = 0; i < num_pages; i++) { pages[i] = alloc_pages_node(cpu_to_node(max(bufb->cpu, 0)), GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0); @@ -97,7 +95,7 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config /* Allocate backend pages array elements */ for (i = 0; i < num_subbuf_alloc; i++) { bufb->array[i] = - lttng_kvzalloc_node(ALIGN( + kvzalloc_node(ALIGN( sizeof(struct lib_ring_buffer_backend_pages) + sizeof(struct lib_ring_buffer_backend_page) * num_pages_per_subbuf, @@ -109,7 +107,7 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config } /* Allocate write-side subbuffer table */ - bufb->buf_wsb = lttng_kvzalloc_node(ALIGN( + bufb->buf_wsb = kvzalloc_node(ALIGN( sizeof(struct lib_ring_buffer_backend_subbuffer) * num_subbuf, 1 << INTERNODE_CACHE_SHIFT), @@ -129,7 +127,7 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config bufb->buf_rsb.id = subbuffer_id(config, 0, 1, 0); /* Allocate subbuffer packet counter table */ - bufb->buf_cnt = lttng_kvzalloc_node(ALIGN( + bufb->buf_cnt = kvzalloc_node(ALIGN( sizeof(struct lib_ring_buffer_backend_counts) * num_subbuf, 1 << INTERNODE_CACHE_SHIFT), @@ -152,25 +150,20 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config } } - /* - * If kmalloc ever uses vmalloc underneath, make sure the buffer pages - * will not fault. - */ - wrapper_vmalloc_sync_all(); wrapper_clear_current_oom_origin(); vfree(pages); return 0; free_wsb: - lttng_kvfree(bufb->buf_wsb); + kvfree(bufb->buf_wsb); free_array: for (i = 0; (i < num_subbuf_alloc && bufb->array[i]); i++) - lttng_kvfree(bufb->array[i]); + kvfree(bufb->array[i]); depopulate: /* Free all allocated pages */ for (i = 0; (i < num_pages && pages[i]); i++) __free_page(pages[i]); - lttng_kvfree(bufb->array); + kvfree(bufb->array); array_error: vfree(pages); pages_error: @@ -201,14 +194,14 @@ void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend *bufb) if (chanb->extra_reader_sb) num_subbuf_alloc++; - lttng_kvfree(bufb->buf_wsb); - lttng_kvfree(bufb->buf_cnt); + kvfree(bufb->buf_wsb); + kvfree(bufb->buf_cnt); for (i = 0; i < num_subbuf_alloc; i++) { for (j = 0; j < bufb->num_pages_per_subbuf; j++) __free_page(pfn_to_page(bufb->array[i]->p[j].pfn)); - lttng_kvfree(bufb->array[i]); + kvfree(bufb->array[i]); } - lttng_kvfree(bufb->array); + kvfree(bufb->array); bufb->allocated = 0; } diff --git a/lib/ringbuffer/ring_buffer_frontend.c b/lib/ringbuffer/ring_buffer_frontend.c index 91cf0ae5..f9397878 100644 --- a/lib/ringbuffer/ring_buffer_frontend.c +++ b/lib/ringbuffer/ring_buffer_frontend.c @@ -51,7 +51,6 @@ #include #include #include -#include /* * Internal structure representing offsets to use at a sub-buffer switch. @@ -134,9 +133,9 @@ void lib_ring_buffer_free(struct lib_ring_buffer *buf) struct channel *chan = buf->backend.chan; lib_ring_buffer_print_errors(chan, buf, buf->backend.cpu); - lttng_kvfree(buf->commit_hot); - lttng_kvfree(buf->commit_cold); - lttng_kvfree(buf->ts_end); + kvfree(buf->commit_hot); + kvfree(buf->commit_cold); + kvfree(buf->ts_end); lib_ring_buffer_backend_free(&buf->backend); } @@ -234,7 +233,7 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, return ret; buf->commit_hot = - lttng_kvzalloc_node(ALIGN(sizeof(*buf->commit_hot) + kvzalloc_node(ALIGN(sizeof(*buf->commit_hot) * chan->backend.num_subbuf, 1 << INTERNODE_CACHE_SHIFT), GFP_KERNEL | __GFP_NOWARN, @@ -245,7 +244,7 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, } buf->commit_cold = - lttng_kvzalloc_node(ALIGN(sizeof(*buf->commit_cold) + kvzalloc_node(ALIGN(sizeof(*buf->commit_cold) * chan->backend.num_subbuf, 1 << INTERNODE_CACHE_SHIFT), GFP_KERNEL | __GFP_NOWARN, @@ -256,7 +255,7 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, } buf->ts_end = - lttng_kvzalloc_node(ALIGN(sizeof(*buf->ts_end) + kvzalloc_node(ALIGN(sizeof(*buf->ts_end) * chan->backend.num_subbuf, 1 << INTERNODE_CACHE_SHIFT), GFP_KERNEL | __GFP_NOWARN, @@ -300,16 +299,15 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, chan->backend.cpumask)); cpumask_set_cpu(cpu, chan->backend.cpumask); } - return 0; /* Error handling */ free_init: - lttng_kvfree(buf->ts_end); + kvfree(buf->ts_end); free_commit_cold: - lttng_kvfree(buf->commit_cold); + kvfree(buf->commit_cold); free_commit: - lttng_kvfree(buf->commit_hot); + kvfree(buf->commit_hot); free_chanbuf: lib_ring_buffer_backend_free(&buf->backend); return ret; diff --git a/lttng-abi.c b/lttng-abi.c index 590945c4..baaf34d3 100644 --- a/lttng-abi.c +++ b/lttng-abi.c @@ -30,7 +30,6 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ #include #include #include @@ -1938,7 +1937,6 @@ int __init lttng_abi_init(void) { int ret = 0; - wrapper_vmalloc_sync_all(); lttng_clock_ref(); ret = lttng_tp_mempool_init(); diff --git a/lttng-context-callstack.c b/lttng-context-callstack.c index 317efdf4..9db3471d 100644 --- a/lttng-context-callstack.c +++ b/lttng-context-callstack.c @@ -46,7 +46,6 @@ #include "lttng-events.h" #include "wrapper/ringbuffer/backend.h" #include "wrapper/ringbuffer/frontend.h" -#include "wrapper/vmalloc.h" #include "lttng-tracer.h" #include "lttng-endian.h" @@ -152,7 +151,6 @@ int __lttng_add_callstack_generic(struct lttng_ctx **ctx, sequence_field->priv = fdata; sequence_field->destroy = lttng_callstack_sequence_destroy; - wrapper_vmalloc_sync_all(); return 0; error_create: diff --git a/lttng-context-cgroup-ns.c b/lttng-context-cgroup-ns.c index a7f4e80f..0c284966 100644 --- a/lttng-context-cgroup-ns.c +++ b/lttng-context-cgroup-ns.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -97,7 +96,6 @@ int lttng_add_cgroup_ns_to_ctx(struct lttng_ctx **ctx) field->record = cgroup_ns_record; field->get_value = cgroup_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_cgroup_ns_to_ctx); diff --git a/lttng-context-cpu-id.c b/lttng-context-cpu-id.c index d3d55b88..18e0508a 100644 --- a/lttng-context-cpu-id.c +++ b/lttng-context-cpu-id.c @@ -12,7 +12,6 @@ #include #include #include -#include #include static @@ -68,7 +67,6 @@ int lttng_add_cpu_id_to_ctx(struct lttng_ctx **ctx) field->record = cpu_id_record; field->get_value = cpu_id_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_cpu_id_to_ctx); diff --git a/lttng-context-egid.c b/lttng-context-egid.c index fcd01461..5dd581ea 100644 --- a/lttng-context-egid.c +++ b/lttng-context-egid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_egid_to_ctx(struct lttng_ctx **ctx) field->record = egid_record; field->get_value = egid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_egid_to_ctx); diff --git a/lttng-context-euid.c b/lttng-context-euid.c index b4b72184..d14f310e 100644 --- a/lttng-context-euid.c +++ b/lttng-context-euid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_euid_to_ctx(struct lttng_ctx **ctx) field->record = euid_record; field->get_value = euid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_euid_to_ctx); diff --git a/lttng-context-gid.c b/lttng-context-gid.c index 586934ed..195efbb0 100644 --- a/lttng-context-gid.c +++ b/lttng-context-gid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_gid_to_ctx(struct lttng_ctx **ctx) field->record = gid_record; field->get_value = gid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_gid_to_ctx); diff --git a/lttng-context-hostname.c b/lttng-context-hostname.c index 0bbb1c78..8462da41 100644 --- a/lttng-context-hostname.c +++ b/lttng-context-hostname.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #define LTTNG_HOSTNAME_CTX_LEN (__NEW_UTS_LEN + 1) @@ -100,7 +99,6 @@ int lttng_add_hostname_to_ctx(struct lttng_ctx **ctx) field->record = hostname_record; field->get_value = hostname_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_hostname_to_ctx); diff --git a/lttng-context-interruptible.c b/lttng-context-interruptible.c index c07cba18..67081db4 100644 --- a/lttng-context-interruptible.c +++ b/lttng-context-interruptible.c @@ -13,7 +13,6 @@ #include #include #include -#include #include /* @@ -75,7 +74,6 @@ int lttng_add_interruptible_to_ctx(struct lttng_ctx **ctx) field->record = interruptible_record; field->get_value = interruptible_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_interruptible_to_ctx); diff --git a/lttng-context-ipc-ns.c b/lttng-context-ipc-ns.c index cccd9766..09d8f2e3 100644 --- a/lttng-context-ipc-ns.c +++ b/lttng-context-ipc-ns.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -96,7 +95,6 @@ int lttng_add_ipc_ns_to_ctx(struct lttng_ctx **ctx) field->record = ipc_ns_record; field->get_value = ipc_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_ipc_ns_to_ctx); diff --git a/lttng-context-migratable.c b/lttng-context-migratable.c index 1a0dc4a1..c8147dd8 100644 --- a/lttng-context-migratable.c +++ b/lttng-context-migratable.c @@ -13,7 +13,6 @@ #include #include #include -#include #include static @@ -68,7 +67,6 @@ int lttng_add_migratable_to_ctx(struct lttng_ctx **ctx) field->record = migratable_record; field->get_value = migratable_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_migratable_to_ctx); diff --git a/lttng-context-mnt-ns.c b/lttng-context-mnt-ns.c index 02940139..72e8efa1 100644 --- a/lttng-context-mnt-ns.c +++ b/lttng-context-mnt-ns.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -98,7 +97,6 @@ int lttng_add_mnt_ns_to_ctx(struct lttng_ctx **ctx) field->record = mnt_ns_record; field->get_value = mnt_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_mnt_ns_to_ctx); diff --git a/lttng-context-need-reschedule.c b/lttng-context-need-reschedule.c index 9a51f0c6..ba30261e 100644 --- a/lttng-context-need-reschedule.c +++ b/lttng-context-need-reschedule.c @@ -13,7 +13,6 @@ #include #include #include -#include #include static @@ -68,7 +67,6 @@ int lttng_add_need_reschedule_to_ctx(struct lttng_ctx **ctx) field->record = need_reschedule_record; field->get_value = need_reschedule_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_need_reschedule_to_ctx); diff --git a/lttng-context-net-ns.c b/lttng-context-net-ns.c index 4584991f..6cdd622a 100644 --- a/lttng-context-net-ns.c +++ b/lttng-context-net-ns.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include @@ -97,7 +96,6 @@ int lttng_add_net_ns_to_ctx(struct lttng_ctx **ctx) field->record = net_ns_record; field->get_value = net_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_net_ns_to_ctx); diff --git a/lttng-context-nice.c b/lttng-context-nice.c index 89283d0f..2eefec34 100644 --- a/lttng-context-nice.c +++ b/lttng-context-nice.c @@ -12,7 +12,6 @@ #include #include #include -#include #include static @@ -68,7 +67,6 @@ int lttng_add_nice_to_ctx(struct lttng_ctx **ctx) field->record = nice_record; field->get_value = nice_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_nice_to_ctx); diff --git a/lttng-context-perf-counters.c b/lttng-context-perf-counters.c index 1c5ab9fb..8625842c 100644 --- a/lttng-context-perf-counters.c +++ b/lttng-context-perf-counters.c @@ -13,9 +13,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -106,7 +106,7 @@ void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field) #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ kfree(field->event_field.name); kfree(field->u.perf_counter->attr); - lttng_kvfree(events); + kvfree(events); kfree(field->u.perf_counter); } @@ -224,7 +224,7 @@ int lttng_add_perf_counter_to_ctx(uint32_t type, int ret; char *name_alloc; - events = lttng_kvzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL); + events = kvzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL); if (!events) return -ENOMEM; @@ -321,7 +321,6 @@ int lttng_add_perf_counter_to_ctx(uint32_t type, field->u.perf_counter = perf_field; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) @@ -359,6 +358,6 @@ name_alloc_error: error_alloc_perf_field: kfree(attr); error_attr: - lttng_kvfree(events); + kvfree(events); return ret; } diff --git a/lttng-context-pid-ns.c b/lttng-context-pid-ns.c index e4d3a3a3..cb1ee0d4 100644 --- a/lttng-context-pid-ns.c +++ b/lttng-context-pid-ns.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -98,7 +97,6 @@ int lttng_add_pid_ns_to_ctx(struct lttng_ctx **ctx) field->record = pid_ns_record; field->get_value = pid_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_pid_ns_to_ctx); diff --git a/lttng-context-pid.c b/lttng-context-pid.c index 3cdc5128..fc7cf255 100644 --- a/lttng-context-pid.c +++ b/lttng-context-pid.c @@ -12,7 +12,6 @@ #include #include #include -#include #include static @@ -68,7 +67,6 @@ int lttng_add_pid_to_ctx(struct lttng_ctx **ctx) field->record = pid_record; field->get_value = pid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_pid_to_ctx); diff --git a/lttng-context-ppid.c b/lttng-context-ppid.c index 8935fad0..bba8f657 100644 --- a/lttng-context-ppid.c +++ b/lttng-context-ppid.c @@ -13,7 +13,6 @@ #include #include #include -#include #include static @@ -90,7 +89,6 @@ int lttng_add_ppid_to_ctx(struct lttng_ctx **ctx) field->record = ppid_record; field->get_value = ppid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_ppid_to_ctx); diff --git a/lttng-context-preemptible.c b/lttng-context-preemptible.c index 6a40f03a..40b2205e 100644 --- a/lttng-context-preemptible.c +++ b/lttng-context-preemptible.c @@ -13,7 +13,6 @@ #include #include #include -#include #include /* @@ -86,7 +85,6 @@ int lttng_add_preemptible_to_ctx(struct lttng_ctx **ctx) field->record = preemptible_record; field->get_value = preemptible_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_preemptible_to_ctx); diff --git a/lttng-context-prio.c b/lttng-context-prio.c index 9cd843d5..23fa9d3a 100644 --- a/lttng-context-prio.c +++ b/lttng-context-prio.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -89,7 +88,6 @@ int lttng_add_prio_to_ctx(struct lttng_ctx **ctx) field->record = prio_record; field->get_value = prio_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_prio_to_ctx); diff --git a/lttng-context-procname.c b/lttng-context-procname.c index 8e4bbf97..40b23dbf 100644 --- a/lttng-context-procname.c +++ b/lttng-context-procname.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -71,7 +70,6 @@ int lttng_add_procname_to_ctx(struct lttng_ctx **ctx) field->record = procname_record; field->get_value = procname_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_procname_to_ctx); diff --git a/lttng-context-sgid.c b/lttng-context-sgid.c index 4f58a14f..fae736da 100644 --- a/lttng-context-sgid.c +++ b/lttng-context-sgid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_sgid_to_ctx(struct lttng_ctx **ctx) field->record = sgid_record; field->get_value = sgid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_sgid_to_ctx); diff --git a/lttng-context-suid.c b/lttng-context-suid.c index 4b8477e5..60b08c43 100644 --- a/lttng-context-suid.c +++ b/lttng-context-suid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_suid_to_ctx(struct lttng_ctx **ctx) field->record = suid_record; field->get_value = suid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_suid_to_ctx); diff --git a/lttng-context-tid.c b/lttng-context-tid.c index 5f8e68fa..48c307b0 100644 --- a/lttng-context-tid.c +++ b/lttng-context-tid.c @@ -12,7 +12,6 @@ #include #include #include -#include #include static @@ -71,7 +70,6 @@ int lttng_add_tid_to_ctx(struct lttng_ctx **ctx) field->record = tid_record; field->get_value = tid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_tid_to_ctx); diff --git a/lttng-context-uid.c b/lttng-context-uid.c index c7a2bb18..0baa2d63 100644 --- a/lttng-context-uid.c +++ b/lttng-context-uid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_uid_to_ctx(struct lttng_ctx **ctx) field->record = uid_record; field->get_value = uid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_uid_to_ctx); diff --git a/lttng-context-user-ns.c b/lttng-context-user-ns.c index ff56f551..bb08f8c1 100644 --- a/lttng-context-user-ns.c +++ b/lttng-context-user-ns.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -82,7 +81,6 @@ int lttng_add_user_ns_to_ctx(struct lttng_ctx **ctx) field->record = user_ns_record; field->get_value = user_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_user_ns_to_ctx); diff --git a/lttng-context-uts-ns.c b/lttng-context-uts-ns.c index 67c42764..c6c6d072 100644 --- a/lttng-context-uts-ns.c +++ b/lttng-context-uts-ns.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -96,7 +95,6 @@ int lttng_add_uts_ns_to_ctx(struct lttng_ctx **ctx) field->record = uts_ns_record; field->get_value = uts_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_uts_ns_to_ctx); diff --git a/lttng-context-vegid.c b/lttng-context-vegid.c index 501b01e6..1330a4f6 100644 --- a/lttng-context-vegid.c +++ b/lttng-context-vegid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_vegid_to_ctx(struct lttng_ctx **ctx) field->record = vegid_record; field->get_value = vegid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vegid_to_ctx); diff --git a/lttng-context-veuid.c b/lttng-context-veuid.c index a8810a77..28b7d3b2 100644 --- a/lttng-context-veuid.c +++ b/lttng-context-veuid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_veuid_to_ctx(struct lttng_ctx **ctx) field->record = veuid_record; field->get_value = veuid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_veuid_to_ctx); diff --git a/lttng-context-vgid.c b/lttng-context-vgid.c index 92ac567b..6f35e840 100644 --- a/lttng-context-vgid.c +++ b/lttng-context-vgid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_vgid_to_ctx(struct lttng_ctx **ctx) field->record = vgid_record; field->get_value = vgid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vgid_to_ctx); diff --git a/lttng-context-vpid.c b/lttng-context-vpid.c index 31a2c15e..4093675e 100644 --- a/lttng-context-vpid.c +++ b/lttng-context-vpid.c @@ -12,7 +12,6 @@ #include #include #include -#include #include static @@ -83,7 +82,6 @@ int lttng_add_vpid_to_ctx(struct lttng_ctx **ctx) field->record = vpid_record; field->get_value = vpid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vpid_to_ctx); diff --git a/lttng-context-vppid.c b/lttng-context-vppid.c index 43bfcc12..f8d66c91 100644 --- a/lttng-context-vppid.c +++ b/lttng-context-vppid.c @@ -13,7 +13,6 @@ #include #include #include -#include #include static @@ -112,7 +111,6 @@ int lttng_add_vppid_to_ctx(struct lttng_ctx **ctx) field->record = vppid_record; field->get_value = vppid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vppid_to_ctx); diff --git a/lttng-context-vsgid.c b/lttng-context-vsgid.c index 20987dde..25221ca6 100644 --- a/lttng-context-vsgid.c +++ b/lttng-context-vsgid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_vsgid_to_ctx(struct lttng_ctx **ctx) field->record = vsgid_record; field->get_value = vsgid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vsgid_to_ctx); diff --git a/lttng-context-vsuid.c b/lttng-context-vsuid.c index 7cefd535..d5677ba9 100644 --- a/lttng-context-vsuid.c +++ b/lttng-context-vsuid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_vsuid_to_ctx(struct lttng_ctx **ctx) field->record = vsuid_record; field->get_value = vsuid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vsuid_to_ctx); diff --git a/lttng-context-vtid.c b/lttng-context-vtid.c index a7da7d2e..dc0e99b9 100644 --- a/lttng-context-vtid.c +++ b/lttng-context-vtid.c @@ -12,7 +12,6 @@ #include #include #include -#include #include static @@ -83,7 +82,6 @@ int lttng_add_vtid_to_ctx(struct lttng_ctx **ctx) field->record = vtid_record; field->get_value = vtid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vtid_to_ctx); diff --git a/lttng-context-vuid.c b/lttng-context-vuid.c index 427ada71..f62964ed 100644 --- a/lttng-context-vuid.c +++ b/lttng-context-vuid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static @@ -70,7 +69,6 @@ int lttng_add_vuid_to_ctx(struct lttng_ctx **ctx) field->record = vuid_record; field->get_value = vuid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vuid_to_ctx); diff --git a/lttng-context.c b/lttng-context.c index 54fa4478..9f841727 100644 --- a/lttng-context.c +++ b/lttng-context.c @@ -11,7 +11,6 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ #include #include @@ -82,12 +81,13 @@ struct lttng_ctx_field *lttng_append_context(struct lttng_ctx **ctx_p) struct lttng_ctx_field *new_fields; ctx->allocated_fields = max_t(size_t, 1, 2 * ctx->allocated_fields); - new_fields = lttng_kvzalloc(ctx->allocated_fields * sizeof(struct lttng_ctx_field), GFP_KERNEL); + new_fields = kvzalloc_node(ctx->allocated_fields * sizeof(struct lttng_ctx_field), + GFP_KERNEL, NUMA_NO_NODE); if (!new_fields) return NULL; if (ctx->fields) memcpy(new_fields, ctx->fields, sizeof(*ctx->fields) * ctx->nr_fields); - lttng_kvfree(ctx->fields); + kvfree(ctx->fields); ctx->fields = new_fields; } field = &ctx->fields[ctx->nr_fields]; @@ -205,7 +205,7 @@ void lttng_destroy_context(struct lttng_ctx *ctx) if (ctx->fields[i].destroy) ctx->fields[i].destroy(&ctx->fields[i]); } - lttng_kvfree(ctx->fields); + kvfree(ctx->fields); kfree(ctx); } diff --git a/lttng-events.c b/lttng-events.c index 1ca4d519..aa0f2e2a 100644 --- a/lttng-events.c +++ b/lttng-events.c @@ -26,11 +26,10 @@ #include #include #include -#include #include #include +#include -#include /* for wrapper_vmalloc_sync_all() */ #include #include #include @@ -129,7 +128,8 @@ struct lttng_session *lttng_session_create(void) int i; mutex_lock(&sessions_mutex); - session = lttng_kvzalloc(sizeof(struct lttng_session), GFP_KERNEL); + session = kvzalloc_node(sizeof(struct lttng_session), GFP_KERNEL, + NUMA_NO_NODE); if (!session) goto err; INIT_LIST_HEAD(&session->chan); @@ -172,7 +172,7 @@ struct lttng_session *lttng_session_create(void) err_free_cache: kfree(metadata_cache); err_free_session: - lttng_kvfree(session); + kvfree(session); err: mutex_unlock(&sessions_mutex); return NULL; @@ -225,7 +225,7 @@ void lttng_session_destroy(struct lttng_session *session) kref_put(&session->metadata_cache->refcount, metadata_cache_destroy); list_del(&session->list); mutex_unlock(&sessions_mutex); - lttng_kvfree(session); + kvfree(session); } int lttng_session_statedump(struct lttng_session *session) @@ -2793,22 +2793,10 @@ end: * @transport: transport structure * * Registers a transport which can be used as output to extract the data out of - * LTTng. The module calling this registration function must ensure that no - * trap-inducing code will be executed by the transport functions. E.g. - * vmalloc_sync_all() must be called between a vmalloc and the moment the memory - * is made visible to the transport function. This registration acts as a - * vmalloc_sync_all. Therefore, only if the module allocates virtual memory - * after its registration must it synchronize the TLBs. + * LTTng. */ void lttng_transport_register(struct lttng_transport *transport) { - /* - * Make sure no page fault can be triggered by the module about to be - * registered. We deal with this here so we don't have to call - * vmalloc_sync_all() in each module's init. - */ - wrapper_vmalloc_sync_all(); - mutex_lock(&sessions_mutex); list_add_tail(&transport->node, <tng_transport_list); mutex_unlock(&sessions_mutex); diff --git a/lttng-ring-buffer-client.h b/lttng-ring-buffer-client.h index f11c606b..05cab588 100644 --- a/lttng-ring-buffer-client.h +++ b/lttng-ring-buffer-client.h @@ -10,7 +10,6 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ #include #include #include @@ -762,11 +761,6 @@ static struct lttng_transport lttng_relay_transport = { static int __init lttng_ring_buffer_client_init(void) { - /* - * This vmalloc sync all also takes care of the lib ring buffer - * vmalloc'd module pages when it is built as a module into LTTng. - */ - wrapper_vmalloc_sync_all(); lttng_transport_register(<tng_relay_transport); return 0; } diff --git a/lttng-ring-buffer-metadata-client.h b/lttng-ring-buffer-metadata-client.h index 4c102975..aef77806 100644 --- a/lttng-ring-buffer-metadata-client.h +++ b/lttng-ring-buffer-metadata-client.h @@ -9,7 +9,6 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ #include #include @@ -423,11 +422,6 @@ static struct lttng_transport lttng_relay_transport = { static int __init lttng_ring_buffer_client_init(void) { - /* - * This vmalloc sync all also takes care of the lib ring buffer - * vmalloc'd module pages when it is built as a module into LTTng. - */ - wrapper_vmalloc_sync_all(); lttng_transport_register(<tng_relay_transport); return 0; } diff --git a/lttng-syscalls.c b/lttng-syscalls.c index a43c475e..3d2025eb 100644 --- a/lttng-syscalls.c +++ b/lttng-syscalls.c @@ -761,8 +761,6 @@ int lttng_syscalls_register(struct lttng_channel *chan, void *filter) struct lttng_kernel_event ev; int ret; - wrapper_vmalloc_sync_all(); - if (!chan->sc_table) { /* create syscall table mapping syscall to events */ chan->sc_table = kzalloc(sizeof(struct lttng_event *) diff --git a/lttng-tracer.h b/lttng-tracer.h index 197e28af..5490d52c 100644 --- a/lttng-tracer.h +++ b/lttng-tracer.h @@ -23,7 +23,6 @@ #include #include -#include #include #include diff --git a/probes/lttng-kprobes.c b/probes/lttng-kprobes.c index 40fe0f68..e3f82ad3 100644 --- a/probes/lttng-kprobes.c +++ b/probes/lttng-kprobes.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -127,13 +126,6 @@ int lttng_kprobes_register(const char *name, event->u.kprobe.kp.offset = offset; event->u.kprobe.kp.addr = (void *) (unsigned long) addr; - /* - * Ensure the memory we just allocated don't trigger page faults. - * Well.. kprobes itself puts the page fault handler on the blacklist, - * but we can never be too careful. - */ - wrapper_vmalloc_sync_all(); - ret = register_kprobe(&event->u.kprobe.kp); if (ret) goto register_error; diff --git a/probes/lttng-kretprobes.c b/probes/lttng-kretprobes.c index 40938f2b..d2082926 100644 --- a/probes/lttng-kretprobes.c +++ b/probes/lttng-kretprobes.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -216,13 +215,6 @@ int lttng_kretprobes_register(const char *name, kref_init(<tng_krp->kref_register); kref_get(<tng_krp->kref_register); /* inc refcount to 2, no overflow. */ - /* - * Ensure the memory we just allocated don't trigger page faults. - * Well.. kprobes itself puts the page fault handler on the blacklist, - * but we can never be too careful. - */ - wrapper_vmalloc_sync_all(); - ret = register_kretprobe(<tng_krp->krp); if (ret) goto register_error; diff --git a/probes/lttng-tracepoint-event-impl.h b/probes/lttng-tracepoint-event-impl.h index 5f77bf4d..cbb9cbae 100644 --- a/probes/lttng-tracepoint-event-impl.h +++ b/probes/lttng-tracepoint-event-impl.h @@ -15,7 +15,6 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ #include #include #include @@ -1414,7 +1413,6 @@ static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = { #ifndef TP_MODULE_NOINIT static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void) { - wrapper_vmalloc_sync_all(); return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM)); } diff --git a/probes/lttng-uprobes.c b/probes/lttng-uprobes.c index 175c0dd9..330fae8a 100644 --- a/probes/lttng-uprobes.c +++ b/probes/lttng-uprobes.c @@ -20,7 +20,6 @@ #include #include #include -#include static int lttng_uprobes_handler_pre(struct uprobe_consumer *uc, struct pt_regs *regs) @@ -160,9 +159,6 @@ int lttng_uprobes_add_callsite(struct lttng_event *event, goto end; } - /* Ensure the memory we just allocated don't trigger page faults. */ - wrapper_vmalloc_sync_all(); - uprobe_handler->event = event; uprobe_handler->up_consumer.handler = lttng_uprobes_handler_pre; diff --git a/probes/lttng.c b/probes/lttng.c index 160e0bcd..7a2a74a1 100644 --- a/probes/lttng.c +++ b/probes/lttng.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #define TP_MODULE_NOAUTOLOAD @@ -116,8 +115,6 @@ int __init lttng_logger_init(void) { int ret = 0; - wrapper_vmalloc_sync_all(); - /* /dev/lttng-logger */ ret = misc_register(&logger_dev); if (ret) { diff --git a/tests/probes/lttng-test.c b/tests/probes/lttng-test.c index cfdacc94..2219f86d 100644 --- a/tests/probes/lttng-test.c +++ b/tests/probes/lttng-test.c @@ -97,7 +97,6 @@ int __init lttng_test_init(void) int ret = 0; (void) wrapper_lttng_fixup_sig(THIS_MODULE); - wrapper_vmalloc_sync_all(); lttng_test_filter_event_dentry = proc_create_data(LTTNG_TEST_FILTER_EVENT_FILE, S_IRUGO | S_IWUGO, NULL, diff --git a/wrapper/vmalloc.h b/wrapper/vmalloc.h deleted file mode 100644 index 80deacfe..00000000 --- a/wrapper/vmalloc.h +++ /dev/null @@ -1,210 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) - * - * wrapper/vmalloc.h - * - * wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when - * available, else we need to have a kernel that exports this function to GPL - * modules. - * - * Copyright (C) 2011-2012 Mathieu Desnoyers - */ - -#ifndef _LTTNG_WRAPPER_VMALLOC_H -#define _LTTNG_WRAPPER_VMALLOC_H - -#include -#include -#include - -#ifdef CONFIG_KALLSYMS - -#include -#include - -static inline -void wrapper_vmalloc_sync_all(void) -{ - void (*vmalloc_sync_all_sym)(void); - - vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all"); - if (vmalloc_sync_all_sym) { - vmalloc_sync_all_sym(); - } else { -#ifdef CONFIG_X86 - /* - * Only x86 needs vmalloc_sync_all to make sure LTTng does not - * trigger recursive page faults. - */ - printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n"); - printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n"); -#endif - } -} -#else - -static inline -void wrapper_vmalloc_sync_all(void) -{ - return vmalloc_sync_all(); -} -#endif - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)) -static inline -void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node) -{ - void *ret; - - ret = kvmalloc_node(size, flags, node); - if (is_vmalloc_addr(ret)) { - /* - * Make sure we don't trigger recursive page faults in the - * tracing fast path. - */ - wrapper_vmalloc_sync_all(); - } - return ret; -} - -static inline -void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node) -{ - return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node); -} - -static inline -void *lttng_kvmalloc(unsigned long size, gfp_t flags) -{ - return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE); -} - -static inline -void *lttng_kvzalloc(unsigned long size, gfp_t flags) -{ - return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE); -} - -static inline -void lttng_kvfree(const void *addr) -{ - kvfree(addr); -} - -#else - -#include - -static inline -void print_vmalloc_node_range_warning(void) -{ - printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n"); - printk_once(KERN_WARNING "Tracer performance will be degraded on NUMA systems.\n"); - printk_once(KERN_WARNING "Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n"); -} - -/* - * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node. - */ -static inline -void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align, - unsigned long start, unsigned long end, gfp_t gfp_mask, - pgprot_t prot, unsigned long vm_flags, int node, - const void *caller) -{ -#ifdef CONFIG_KALLSYMS - /* - * If we have KALLSYMS, get * __vmalloc_node_range which is not exported. - */ - void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align, - unsigned long start, unsigned long end, gfp_t gfp_mask, - pgprot_t prot, unsigned long vm_flags, int node, - const void *caller); - - lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range"); - if (lttng__vmalloc_node_range) - return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot, - vm_flags, node, caller); -#endif - if (node != NUMA_NO_NODE) - print_vmalloc_node_range_warning(); - return __vmalloc(size, gfp_mask, prot); -} - -/** - * lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon - * failure, fall back to non-contiguous (vmalloc) allocation. - * @size: size of the request. - * @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL. - * - * Uses kmalloc to get the memory but if the allocation fails then falls back - * to the vmalloc allocator. Use lttng_kvfree to free the memory. - * - * Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported - */ -static inline -void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node) -{ - void *ret; - - /* - * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables) - * so the given set of flags has to be compatible. - */ - WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL); - - /* - * If the allocation fits in a single page, do not fallback. - */ - if (size <= PAGE_SIZE) { - return kmalloc_node(size, flags, node); - } - - /* - * Make sure that larger requests are not too disruptive - no OOM - * killer and no allocation failure warnings as we have a fallback - */ - ret = kmalloc_node(size, flags | __GFP_NOWARN | __GFP_NORETRY, node); - if (!ret) { - ret = __lttng_vmalloc_node_range(size, 1, - VMALLOC_START, VMALLOC_END, - flags | __GFP_HIGHMEM, PAGE_KERNEL, 0, - node, __builtin_return_address(0)); - /* - * Make sure we don't trigger recursive page faults in the - * tracing fast path. - */ - wrapper_vmalloc_sync_all(); - } - return ret; -} - -static inline -void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node) -{ - return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node); -} - -static inline -void *lttng_kvmalloc(unsigned long size, gfp_t flags) -{ - return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE); -} - -static inline -void *lttng_kvzalloc(unsigned long size, gfp_t flags) -{ - return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE); -} - -static inline -void lttng_kvfree(const void *addr) -{ - if (is_vmalloc_addr(addr)) { - vfree(addr); - } else { - kfree(addr); - } -} -#endif - -#endif /* _LTTNG_WRAPPER_VMALLOC_H */ -- 2.34.1