[lttng-modules.git] / wrapper / vmalloc.h

/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
 *
 * wrapper/vmalloc.h
 *
 * wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
 * available, else we need to have a kernel that exports this function to GPL
 * modules.
 *
 * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 */

#ifndef _LTTNG_WRAPPER_VMALLOC_H
#define _LTTNG_WRAPPER_VMALLOC_H

#include <linux/version.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>

#ifdef CONFIG_KALLSYMS

#include <linux/kallsyms.h>
#include <wrapper/kallsyms.h>
#include <lttng-kernel-version.h>

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,8,0))

/*
 * wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
 * are now synchronized when they are created or torn down.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{}

#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) \
  || LTTNG_KERNEL_RANGE(5,5,12, 5,6,0)            \
  || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0)            \
  || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0)            \
  || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0)         \
  || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0)         \
  || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0)          \
  || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0))	  \
  || LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,16,0,0) \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0)   \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	void (*vmalloc_sync_mappings_sym)(void);

	vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
	if (vmalloc_sync_mappings_sym) {
		vmalloc_sync_mappings_sym();
	} else {
#ifdef CONFIG_X86
		/*
		 * Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
		 * trigger recursive page faults.
		 */
		printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
		printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
#endif
	}
}

#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */

/*
 * Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.7.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{
	void (*vmalloc_sync_all_sym)(void);

	vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
	if (vmalloc_sync_all_sym) {
		vmalloc_sync_all_sym();
	} else {
#ifdef CONFIG_X86
		/*
		 * Only x86 needs vmalloc_sync_all to make sure LTTng does not
		 * trigger recursive page faults.
		 */
		printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n");
		printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
#endif
	}
}

#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */

#else

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) \
  || LTTNG_KERNEL_RANGE(5,5,12, 5,6,0)            \
  || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0)            \
  || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0)            \
  || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0)         \
  || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0)         \
  || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0)          \
  || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0))	  \
  || LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,18,0,0) \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0)   \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	return vmalloc_sync_mappings();
}

#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	return vmalloc_sync_all();
}

#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */

#endif

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
static inline
void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
{
	void *ret;

	ret = kvmalloc_node(size, flags, node);
	if (is_vmalloc_addr(ret)) {
		/*
		 * Make sure we don't trigger recursive page faults in the
		 * tracing fast path.
		 */
		wrapper_vmalloc_sync_mappings();
	}
	return ret;
}

static inline
void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
{
	return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node);
}

static inline
void *lttng_kvmalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void *lttng_kvzalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void lttng_kvfree(const void *addr)
{
	kvfree(addr);
}

#else

#include <linux/slab.h>

static inline
void print_vmalloc_node_range_warning(void)
{
	printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n");
	printk_once(KERN_WARNING "Tracer performance will be degraded on NUMA systems.\n");
	printk_once(KERN_WARNING "Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
}

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				vm_flags, node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/**
 * lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
 * failure, fall back to non-contiguous (vmalloc) allocation.
 * @size: size of the request.
 * @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
 *
 * Uses kmalloc to get the memory but if the allocation fails then falls back
 * to the vmalloc allocator. Use lttng_kvfree to free the memory.
 *
 * Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
 */
static inline
void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
{
	void *ret;

	/*
	 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
	 * so the given set of flags has to be compatible.
	 */
	WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);

	/*
	 * If the allocation fits in a single page, do not fallback.
	 */
	if (size <= PAGE_SIZE) {
		return kmalloc_node(size, flags, node);
	}

	/*
	 * Make sure that larger requests are not too disruptive - no OOM
	 * killer and no allocation failure warnings as we have a fallback
	 */
	ret = kmalloc_node(size, flags | __GFP_NOWARN | __GFP_NORETRY, node);
	if (!ret) {
		ret = __lttng_vmalloc_node_range(size, 1,
				VMALLOC_START, VMALLOC_END,
				flags | __GFP_HIGHMEM, PAGE_KERNEL, 0,
				node, __builtin_return_address(0));
		/*
		 * Make sure we don't trigger recursive page faults in the
		 * tracing fast path.
		 */
		wrapper_vmalloc_sync_mappings();
	}
	return ret;
}

static inline
void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
{
	return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node);
}

static inline
void *lttng_kvmalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void *lttng_kvzalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void lttng_kvfree(const void *addr)
{
	if (is_vmalloc_addr(addr)) {
		vfree(addr);
	} else {
		kfree(addr);
	}
}
#endif

#endif /* _LTTNG_WRAPPER_VMALLOC_H */
Commit	Line	Data
9f36eaed MJ	1	/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
9f36eaed MJ	2	*
886d51a3	3	* wrapper/vmalloc.h
6d2a620c MD	4	*
	5	* wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
	6	* available, else we need to have a kernel that exports this function to GPL
	7	* modules.
	8	*
886d51a3	9	* Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6d2a620c MD	10	*/
6d2a620c MD	11
9f36eaed MJ	12	#ifndef _LTTNG_WRAPPER_VMALLOC_H
	13	#define _LTTNG_WRAPPER_VMALLOC_H
	14
48f5e0b5 MJ	15	#include <linux/version.h>
48f5e0b5 MJ	16	#include <linux/vmalloc.h>
01ab5113	17	#include <linux/mm.h>
48f5e0b5	18
6d2a620c MD	19	#ifdef CONFIG_KALLSYMS
	20
	21	#include <linux/kallsyms.h>
5a2f5e92	22	#include <wrapper/kallsyms.h>
f419315f	23	#include <lttng-kernel-version.h>
6d2a620c	24
11441f8f MJ	25	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,8,0))
	26
	27	/*
	28	* wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
	29	* are now synchronized when they are created or torn down.
	30	*/
	31	static inline
	32	void wrapper_vmalloc_sync_mappings(void)
	33	{}
	34
	35	#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) \
28926981	36	\|\| LTTNG_KERNEL_RANGE(5,5,12, 5,6,0) \
f419315f OP	37	\|\| LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
	38	\|\| LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
	39	\|\| LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
	40	\|\| LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
	41	\|\| LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
64d128d7 SB	42	\|\| LTTNG_KERNEL_RANGE(4,4,218, 4,5,0)) \
	43	\|\| LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,16,0,0) \
	44	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0) \
	45	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)
ea53823c MD	46
	47	static inline
	48	void wrapper_vmalloc_sync_mappings(void)
	49	{
	50	void (*vmalloc_sync_mappings_sym)(void);
	51
	52	vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
	53	if (vmalloc_sync_mappings_sym) {
	54	vmalloc_sync_mappings_sym();
	55	} else {
	56	#ifdef CONFIG_X86
	57	/*
	58	* Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
	59	* trigger recursive page faults.
	60	*/
	61	printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
	62	printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
	63	#endif
	64	}
	65	}
	66
23cfd7b5	67	#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
ea53823c MD	68
	69	/*
	70	* Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.7.
	71	*/
6d2a620c	72	static inline
ea53823c	73	void wrapper_vmalloc_sync_mappings(void)
6d2a620c MD	74	{
	75	void (*vmalloc_sync_all_sym)(void);
	76
c539a324	77	vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
6d2a620c MD	78	if (vmalloc_sync_all_sym) {
	79	vmalloc_sync_all_sym();
	80	} else {
	81	#ifdef CONFIG_X86
	82	/*
	83	* Only x86 needs vmalloc_sync_all to make sure LTTng does not
	84	* trigger recursive page faults.
	85	*/
e36de50d MD	86	printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n");
e36de50d MD	87	printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
6d2a620c MD	88	#endif
	89	}
	90	}
ea53823c	91
23cfd7b5	92	#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
ea53823c	93
6d2a620c MD	94	#else
6d2a620c MD	95
f419315f	96	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) \
28926981	97	\|\| LTTNG_KERNEL_RANGE(5,5,12, 5,6,0) \
f419315f OP	98	\|\| LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
	99	\|\| LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
	100	\|\| LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
	101	\|\| LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
	102	\|\| LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
64d128d7 SB	103	\|\| LTTNG_KERNEL_RANGE(4,4,218, 4,5,0)) \
	104	\|\| LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,18,0,0) \
	105	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0) \
	106	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)
ea53823c	107
6d2a620c	108	static inline
ea53823c MD	109	void wrapper_vmalloc_sync_mappings(void)
	110	{
	111	return vmalloc_sync_mappings();
	112	}
	113
23cfd7b5	114	#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
ea53823c MD	115
	116	static inline
	117	void wrapper_vmalloc_sync_mappings(void)
6d2a620c MD	118	{
	119	return vmalloc_sync_all();
	120	}
ea53823c	121
23cfd7b5	122	#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)) */
ea53823c	123
6d2a620c	124	#endif
b13f3ebe	125
48f5e0b5 MJ	126	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
	127	static inline
	128	void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
	129	{
	130	void *ret;
	131
	132	ret = kvmalloc_node(size, flags, node);
	133	if (is_vmalloc_addr(ret)) {
	134	/*
	135	* Make sure we don't trigger recursive page faults in the
	136	* tracing fast path.
	137	*/
ea53823c	138	wrapper_vmalloc_sync_mappings();
48f5e0b5 MJ	139	}
	140	return ret;
	141	}
	142
	143	static inline
	144	void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
	145	{
	146	return lttng_kvmalloc_node(size, flags \| __GFP_ZERO, node);
	147	}
	148
	149	static inline
	150	void *lttng_kvmalloc(unsigned long size, gfp_t flags)
	151	{
	152	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
	153	}
	154
	155	static inline
	156	void *lttng_kvzalloc(unsigned long size, gfp_t flags)
	157	{
	158	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
	159	}
	160
	161	static inline
	162	void lttng_kvfree(const void *addr)
	163	{
	164	kvfree(addr);
	165	}
	166
	167	#else
	168
	169	#include <linux/slab.h>
48f5e0b5	170
20eb87c9 MD	171	static inline
	172	void print_vmalloc_node_range_warning(void)
	173	{
	174	printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n");
	175	printk_once(KERN_WARNING "Tracer performance will be degraded on NUMA systems.\n");
	176	printk_once(KERN_WARNING "Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
	177	}
	178
48f5e0b5 MJ	179	/*
	180	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	181	*/
	182	static inline
20eb87c9 MD	183	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	184	unsigned long start, unsigned long end, gfp_t gfp_mask,
	185	pgprot_t prot, unsigned long vm_flags, int node,
	186	const void *caller)
48f5e0b5	187	{
48f5e0b5 MJ	188	#ifdef CONFIG_KALLSYMS
48f5e0b5 MJ	189	/*
20eb87c9	190	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
48f5e0b5	191	*/
20eb87c9 MD	192	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	193	unsigned long start, unsigned long end, gfp_t gfp_mask,
	194	pgprot_t prot, unsigned long vm_flags, int node,
	195	const void *caller);
	196
	197	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	198	if (lttng__vmalloc_node_range)
	199	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	200	vm_flags, node, caller);
48f5e0b5	201	#endif
20eb87c9 MD	202	if (node != NUMA_NO_NODE)
	203	print_vmalloc_node_range_warning();
	204	return __vmalloc(size, gfp_mask, prot);
48f5e0b5 MJ	205	}
	206
	207	/**
	208	* lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
	209	* failure, fall back to non-contiguous (vmalloc) allocation.
	210	* @size: size of the request.
	211	* @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
	212	*
	213	* Uses kmalloc to get the memory but if the allocation fails then falls back
	214	* to the vmalloc allocator. Use lttng_kvfree to free the memory.
	215	*
	216	* Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
	217	*/
	218	static inline
	219	void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
	220	{
	221	void *ret;
	222
	223	/*
	224	* vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
	225	* so the given set of flags has to be compatible.
	226	*/
	227	WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);
	228
	229	/*
	230	* If the allocation fits in a single page, do not fallback.
	231	*/
	232	if (size <= PAGE_SIZE) {
	233	return kmalloc_node(size, flags, node);
	234	}
	235
	236	/*
	237	* Make sure that larger requests are not too disruptive - no OOM
	238	* killer and no allocation failure warnings as we have a fallback
	239	*/
	240	ret = kmalloc_node(size, flags \| __GFP_NOWARN \| __GFP_NORETRY, node);
	241	if (!ret) {
20eb87c9 MD	242	ret = __lttng_vmalloc_node_range(size, 1,
	243	VMALLOC_START, VMALLOC_END,
	244	flags \| __GFP_HIGHMEM, PAGE_KERNEL, 0,
	245	node, __builtin_return_address(0));
48f5e0b5 MJ	246	/*
	247	* Make sure we don't trigger recursive page faults in the
	248	* tracing fast path.
	249	*/
4d1817c1	250	wrapper_vmalloc_sync_mappings();
48f5e0b5 MJ	251	}
	252	return ret;
	253	}
	254
	255	static inline
	256	void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
	257	{
	258	return lttng_kvmalloc_node(size, flags \| __GFP_ZERO, node);
	259	}
	260
	261	static inline
	262	void *lttng_kvmalloc(unsigned long size, gfp_t flags)
	263	{
	264	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
	265	}
	266
	267	static inline
	268	void *lttng_kvzalloc(unsigned long size, gfp_t flags)
	269	{
	270	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
	271	}
	272
	273	static inline
	274	void lttng_kvfree(const void *addr)
	275	{
	276	if (is_vmalloc_addr(addr)) {
	277	vfree(addr);
	278	} else {
	279	kfree(addr);
	280	}
	281	}
	282	#endif
	283
a90917c3	284	#endif /* _LTTNG_WRAPPER_VMALLOC_H */