From da0fcb1497ff2437407883647a8a0bba12bd0f91 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 5 May 2020 13:38:31 -0400 Subject: [PATCH] Update for kernel 5.7: use vmalloc_sync_mappings on kernels >= 5.7 Signed-off-by: Mathieu Desnoyers --- lib/ringbuffer/ring_buffer_backend.c | 4 +-- lttng-abi.c | 4 +-- lttng-context-callstack.c | 2 +- lttng-context-cpu-id.c | 2 +- lttng-context-hostname.c | 2 +- lttng-context-interruptible.c | 2 +- lttng-context-migratable.c | 2 +- lttng-context-need-reschedule.c | 2 +- lttng-context-nice.c | 2 +- lttng-context-perf-counters.c | 2 +- lttng-context-pid.c | 2 +- lttng-context-ppid.c | 2 +- lttng-context-preemptible.c | 2 +- lttng-context-prio.c | 2 +- lttng-context-procname.c | 2 +- lttng-context-tid.c | 2 +- lttng-context-vpid.c | 2 +- lttng-context-vppid.c | 2 +- lttng-context-vtid.c | 2 +- lttng-context.c | 2 +- lttng-events.c | 10 +++--- lttng-ring-buffer-client.h | 4 +-- lttng-ring-buffer-metadata-client.h | 4 +-- lttng-syscalls.c | 2 +- probes/lttng-kprobes.c | 2 +- probes/lttng-kretprobes.c | 2 +- probes/lttng-tracepoint-event-impl.h | 4 +-- probes/lttng-uprobes.c | 2 +- probes/lttng.c | 2 +- tests/probes/lttng-test.c | 2 +- wrapper/vmalloc.h | 49 ++++++++++++++++++++++++++-- 31 files changed, 85 insertions(+), 42 deletions(-) diff --git a/lib/ringbuffer/ring_buffer_backend.c b/lib/ringbuffer/ring_buffer_backend.c index d4bec25f..d232b7f2 100644 --- a/lib/ringbuffer/ring_buffer_backend.c +++ b/lib/ringbuffer/ring_buffer_backend.c @@ -17,7 +17,7 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -156,7 +156,7 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config * If kmalloc ever uses vmalloc underneath, make sure the buffer pages * will not fault. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); wrapper_clear_current_oom_origin(); vfree(pages); return 0; diff --git a/lttng-abi.c b/lttng-abi.c index 2c6d6623..816e6fdd 100644 --- a/lttng-abi.c +++ b/lttng-abi.c @@ -30,7 +30,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -1832,7 +1832,7 @@ int __init lttng_abi_init(void) { int ret = 0; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_clock_ref(); ret = lttng_tp_mempool_init(); diff --git a/lttng-context-callstack.c b/lttng-context-callstack.c index 195990a6..ba35a394 100644 --- a/lttng-context-callstack.c +++ b/lttng-context-callstack.c @@ -141,7 +141,7 @@ int __lttng_add_callstack_generic(struct lttng_ctx **ctx, field->record = lttng_callstack_record; field->priv = fdata; field->destroy = lttng_callstack_destroy; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; error_create: diff --git a/lttng-context-cpu-id.c b/lttng-context-cpu-id.c index 37782251..6e2f3f45 100644 --- a/lttng-context-cpu-id.c +++ b/lttng-context-cpu-id.c @@ -68,7 +68,7 @@ int lttng_add_cpu_id_to_ctx(struct lttng_ctx **ctx) field->record = cpu_id_record; field->get_value = cpu_id_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_cpu_id_to_ctx); diff --git a/lttng-context-hostname.c b/lttng-context-hostname.c index e485aa97..17dbd577 100644 --- a/lttng-context-hostname.c +++ b/lttng-context-hostname.c @@ -101,7 +101,7 @@ int lttng_add_hostname_to_ctx(struct lttng_ctx **ctx) field->record = hostname_record; field->get_value = hostname_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_hostname_to_ctx); diff --git a/lttng-context-interruptible.c b/lttng-context-interruptible.c index f3da1990..8bd279a3 100644 --- a/lttng-context-interruptible.c +++ b/lttng-context-interruptible.c @@ -75,7 +75,7 @@ int lttng_add_interruptible_to_ctx(struct lttng_ctx **ctx) field->record = interruptible_record; field->get_value = interruptible_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_interruptible_to_ctx); diff --git a/lttng-context-migratable.c b/lttng-context-migratable.c index e3d8ede5..65154b8b 100644 --- a/lttng-context-migratable.c +++ b/lttng-context-migratable.c @@ -68,7 +68,7 @@ int lttng_add_migratable_to_ctx(struct lttng_ctx **ctx) field->record = migratable_record; field->get_value = migratable_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_migratable_to_ctx); diff --git a/lttng-context-need-reschedule.c b/lttng-context-need-reschedule.c index f29fb28d..94efaa5c 100644 --- a/lttng-context-need-reschedule.c +++ b/lttng-context-need-reschedule.c @@ -68,7 +68,7 @@ int lttng_add_need_reschedule_to_ctx(struct lttng_ctx **ctx) field->record = need_reschedule_record; field->get_value = need_reschedule_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_need_reschedule_to_ctx); diff --git a/lttng-context-nice.c b/lttng-context-nice.c index 563e8058..acaba855 100644 --- a/lttng-context-nice.c +++ b/lttng-context-nice.c @@ -68,7 +68,7 @@ int lttng_add_nice_to_ctx(struct lttng_ctx **ctx) field->record = nice_record; field->get_value = nice_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_nice_to_ctx); diff --git a/lttng-context-perf-counters.c b/lttng-context-perf-counters.c index 43c4077b..6b04d123 100644 --- a/lttng-context-perf-counters.c +++ b/lttng-context-perf-counters.c @@ -321,7 +321,7 @@ int lttng_add_perf_counter_to_ctx(uint32_t type, field->u.perf_counter = perf_field; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) diff --git a/lttng-context-pid.c b/lttng-context-pid.c index b3caabb4..326eabe8 100644 --- a/lttng-context-pid.c +++ b/lttng-context-pid.c @@ -68,7 +68,7 @@ int lttng_add_pid_to_ctx(struct lttng_ctx **ctx) field->record = pid_record; field->get_value = pid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_pid_to_ctx); diff --git a/lttng-context-ppid.c b/lttng-context-ppid.c index ef84ad11..ff25288f 100644 --- a/lttng-context-ppid.c +++ b/lttng-context-ppid.c @@ -90,7 +90,7 @@ int lttng_add_ppid_to_ctx(struct lttng_ctx **ctx) field->record = ppid_record; field->get_value = ppid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_ppid_to_ctx); diff --git a/lttng-context-preemptible.c b/lttng-context-preemptible.c index 328f2a43..867805b7 100644 --- a/lttng-context-preemptible.c +++ b/lttng-context-preemptible.c @@ -86,7 +86,7 @@ int lttng_add_preemptible_to_ctx(struct lttng_ctx **ctx) field->record = preemptible_record; field->get_value = preemptible_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_preemptible_to_ctx); diff --git a/lttng-context-prio.c b/lttng-context-prio.c index b5d21e7e..367f7bd6 100644 --- a/lttng-context-prio.c +++ b/lttng-context-prio.c @@ -89,7 +89,7 @@ int lttng_add_prio_to_ctx(struct lttng_ctx **ctx) field->record = prio_record; field->get_value = prio_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_prio_to_ctx); diff --git a/lttng-context-procname.c b/lttng-context-procname.c index 54007d15..8f18ca2c 100644 --- a/lttng-context-procname.c +++ b/lttng-context-procname.c @@ -72,7 +72,7 @@ int lttng_add_procname_to_ctx(struct lttng_ctx **ctx) field->record = procname_record; field->get_value = procname_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_procname_to_ctx); diff --git a/lttng-context-tid.c b/lttng-context-tid.c index d2a20e60..f6defc46 100644 --- a/lttng-context-tid.c +++ b/lttng-context-tid.c @@ -71,7 +71,7 @@ int lttng_add_tid_to_ctx(struct lttng_ctx **ctx) field->record = tid_record; field->get_value = tid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_tid_to_ctx); diff --git a/lttng-context-vpid.c b/lttng-context-vpid.c index 982ce558..b5b6ce0a 100644 --- a/lttng-context-vpid.c +++ b/lttng-context-vpid.c @@ -83,7 +83,7 @@ int lttng_add_vpid_to_ctx(struct lttng_ctx **ctx) field->record = vpid_record; field->get_value = vpid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vpid_to_ctx); diff --git a/lttng-context-vppid.c b/lttng-context-vppid.c index 66d63824..347ac634 100644 --- a/lttng-context-vppid.c +++ b/lttng-context-vppid.c @@ -112,7 +112,7 @@ int lttng_add_vppid_to_ctx(struct lttng_ctx **ctx) field->record = vppid_record; field->get_value = vppid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vppid_to_ctx); diff --git a/lttng-context-vtid.c b/lttng-context-vtid.c index 8fd68a8c..e81ba0e2 100644 --- a/lttng-context-vtid.c +++ b/lttng-context-vtid.c @@ -83,7 +83,7 @@ int lttng_add_vtid_to_ctx(struct lttng_ctx **ctx) field->record = vtid_record; field->get_value = vtid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vtid_to_ctx); diff --git a/lttng-context.c b/lttng-context.c index 869496d8..ce11f565 100644 --- a/lttng-context.c +++ b/lttng-context.c @@ -11,7 +11,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include diff --git a/lttng-events.c b/lttng-events.c index 6233c75c..d78e8efa 100644 --- a/lttng-events.c +++ b/lttng-events.c @@ -30,7 +30,7 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -2707,9 +2707,9 @@ end: * Registers a transport which can be used as output to extract the data out of * LTTng. The module calling this registration function must ensure that no * trap-inducing code will be executed by the transport functions. E.g. - * vmalloc_sync_all() must be called between a vmalloc and the moment the memory + * vmalloc_sync_mappings() must be called between a vmalloc and the moment the memory * is made visible to the transport function. This registration acts as a - * vmalloc_sync_all. Therefore, only if the module allocates virtual memory + * vmalloc_sync_mappings. Therefore, only if the module allocates virtual memory * after its registration must it synchronize the TLBs. */ void lttng_transport_register(struct lttng_transport *transport) @@ -2717,9 +2717,9 @@ void lttng_transport_register(struct lttng_transport *transport) /* * Make sure no page fault can be triggered by the module about to be * registered. We deal with this here so we don't have to call - * vmalloc_sync_all() in each module's init. + * vmalloc_sync_mappings() in each module's init. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); mutex_lock(&sessions_mutex); list_add_tail(&transport->node, <tng_transport_list); diff --git a/lttng-ring-buffer-client.h b/lttng-ring-buffer-client.h index d5c512c5..ff628308 100644 --- a/lttng-ring-buffer-client.h +++ b/lttng-ring-buffer-client.h @@ -10,7 +10,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -766,7 +766,7 @@ static int __init lttng_ring_buffer_client_init(void) * This vmalloc sync all also takes care of the lib ring buffer * vmalloc'd module pages when it is built as a module into LTTng. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_transport_register(<tng_relay_transport); return 0; } diff --git a/lttng-ring-buffer-metadata-client.h b/lttng-ring-buffer-metadata-client.h index 17ffd759..a098b8d8 100644 --- a/lttng-ring-buffer-metadata-client.h +++ b/lttng-ring-buffer-metadata-client.h @@ -9,7 +9,7 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include @@ -428,7 +428,7 @@ static int __init lttng_ring_buffer_client_init(void) * This vmalloc sync all also takes care of the lib ring buffer * vmalloc'd module pages when it is built as a module into LTTng. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_transport_register(<tng_relay_transport); return 0; } diff --git a/lttng-syscalls.c b/lttng-syscalls.c index fb14f6b0..97f1ba94 100644 --- a/lttng-syscalls.c +++ b/lttng-syscalls.c @@ -760,7 +760,7 @@ int lttng_syscalls_register(struct lttng_channel *chan, void *filter) struct lttng_kernel_event ev; int ret; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); if (!chan->sc_table) { /* create syscall table mapping syscall to events */ diff --git a/probes/lttng-kprobes.c b/probes/lttng-kprobes.c index c0a15e47..a44eaa11 100644 --- a/probes/lttng-kprobes.c +++ b/probes/lttng-kprobes.c @@ -132,7 +132,7 @@ int lttng_kprobes_register(const char *name, * Well.. kprobes itself puts the page fault handler on the blacklist, * but we can never be too careful. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); ret = register_kprobe(&event->u.kprobe.kp); if (ret) diff --git a/probes/lttng-kretprobes.c b/probes/lttng-kretprobes.c index 4b18d460..ab98ff22 100644 --- a/probes/lttng-kretprobes.c +++ b/probes/lttng-kretprobes.c @@ -221,7 +221,7 @@ int lttng_kretprobes_register(const char *name, * Well.. kprobes itself puts the page fault handler on the blacklist, * but we can never be too careful. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); ret = register_kretprobe(<tng_krp->krp); if (ret) diff --git a/probes/lttng-tracepoint-event-impl.h b/probes/lttng-tracepoint-event-impl.h index 3fe9d995..0ab3bd01 100644 --- a/probes/lttng-tracepoint-event-impl.h +++ b/probes/lttng-tracepoint-event-impl.h @@ -15,7 +15,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -1352,7 +1352,7 @@ static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = { #ifndef TP_MODULE_NOINIT static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void) { - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM)); } diff --git a/probes/lttng-uprobes.c b/probes/lttng-uprobes.c index 64d8237c..bc101289 100644 --- a/probes/lttng-uprobes.c +++ b/probes/lttng-uprobes.c @@ -161,7 +161,7 @@ int lttng_uprobes_add_callsite(struct lttng_event *event, } /* Ensure the memory we just allocated don't trigger page faults. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); uprobe_handler->event = event; uprobe_handler->up_consumer.handler = lttng_uprobes_handler_pre; diff --git a/probes/lttng.c b/probes/lttng.c index 383202c6..05bc1388 100644 --- a/probes/lttng.c +++ b/probes/lttng.c @@ -116,7 +116,7 @@ int __init lttng_logger_init(void) { int ret = 0; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); /* /dev/lttng-logger */ ret = misc_register(&logger_dev); diff --git a/tests/probes/lttng-test.c b/tests/probes/lttng-test.c index f2d607a1..c728bed5 100644 --- a/tests/probes/lttng-test.c +++ b/tests/probes/lttng-test.c @@ -98,7 +98,7 @@ int __init lttng_test_init(void) int ret = 0; (void) wrapper_lttng_fixup_sig(THIS_MODULE); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_test_filter_event_dentry = proc_create_data(LTTNG_TEST_FILTER_EVENT_FILE, S_IRUGO | S_IWUGO, NULL, diff --git a/wrapper/vmalloc.h b/wrapper/vmalloc.h index 0c3eb86b..54715b57 100644 --- a/wrapper/vmalloc.h +++ b/wrapper/vmalloc.h @@ -21,8 +21,35 @@ #include #include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) + +static inline +void wrapper_vmalloc_sync_mappings(void) +{ + void (*vmalloc_sync_mappings_sym)(void); + + vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings"); + if (vmalloc_sync_mappings_sym) { + vmalloc_sync_mappings_sym(); + } else { +#ifdef CONFIG_X86 + /* + * Only x86 needs vmalloc_sync_mappings to make sure LTTng does not + * trigger recursive page faults. + */ + printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n"); + printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n"); +#endif + } +} + +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + +/* + * Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.7. + */ static inline -void wrapper_vmalloc_sync_all(void) +void wrapper_vmalloc_sync_mappings(void) { void (*vmalloc_sync_all_sym)(void); @@ -40,13 +67,29 @@ void wrapper_vmalloc_sync_all(void) #endif } } + +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + #else +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) + +static inline +void wrapper_vmalloc_sync_mappings(void) +{ + return vmalloc_sync_mappings(); +} + +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + static inline -void wrapper_vmalloc_sync_all(void) +void wrapper_vmalloc_sync_mappings(void) { return vmalloc_sync_all(); } + +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + #endif #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)) @@ -61,7 +104,7 @@ void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node) * Make sure we don't trigger recursive page faults in the * tracing fast path. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); } return ret; } -- 2.34.1