[lttng-modules.git] / include / wrapper / vmalloc.h

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * wrapper/vmalloc.h
 *
 * wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
 * available, else we need to have a kernel that exports this function to GPL
 * modules.
 *
 * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 */

#ifndef _LTTNG_WRAPPER_VMALLOC_H
#define _LTTNG_WRAPPER_VMALLOC_H

#include <lttng/kernel-version.h>
#include <linux/vmalloc.h>
#include <linux/slab.h>
#include <linux/mm.h>

#ifdef CONFIG_KALLSYMS

#include <linux/kallsyms.h>
#include <wrapper/kallsyms.h>
#include <lttng/kernel-version.h>

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))

/*
 * wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
 * are now synchronized when they are created or torn down.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
  || LTTNG_KERNEL_RANGE(5,5,12, 5,6,0)            \
  || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0)            \
  || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0)            \
  || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0)         \
  || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0)         \
  || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0)          \
  || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0)		  \
  || LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,16,0,0) \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0)   \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)   \
  || LTTNG_RHEL_KERNEL_RANGE(4,18,0,240,0,0, 4,19,0,0,0,0))

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	void (*vmalloc_sync_mappings_sym)(void);

	vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
	if (vmalloc_sync_mappings_sym) {
		vmalloc_sync_mappings_sym();
	} else {
#ifdef CONFIG_X86
		/*
		 * Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
		 * trigger recursive page faults.
		 */
		printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
		printk_once(KERN_WARNING "LTTng: Page fault handler and NMI tracing might trigger faults.\n");
#endif
	}
}

/*
 * Canary function to check for 'vmalloc_sync_mappings()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   void vmalloc_sync_mappings(void);
 */
static inline
void __canary__vmalloc_sync_mappings(void)
{
	vmalloc_sync_mappings();
}

#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

/*
 * Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.6.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{
	void (*vmalloc_sync_all_sym)(void);

	vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
	if (vmalloc_sync_all_sym) {
		vmalloc_sync_all_sym();
	} else {
#ifdef CONFIG_X86
		/*
		 * Only x86 needs vmalloc_sync_all to make sure LTTng does not
		 * trigger recursive page faults.
		 */
		printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n");
		printk_once(KERN_WARNING "LTTng: Page fault handler and NMI tracing might trigger faults.\n");
#endif
	}
}

/*
 * Canary function to check for 'vmalloc_sync_all()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   void vmalloc_sync_all(void);
 */
static inline
void __canary__vmalloc_sync_all(void)
{
	vmalloc_sync_all();
}

#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

#else /* CONFIG_KALLSYMS */

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))

/*
 * wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
 * are now synchronized when they are created or torn down.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
  || LTTNG_KERNEL_RANGE(5,5,12, 5,6,0)            \
  || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0)            \
  || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0)            \
  || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0)         \
  || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0)         \
  || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0)          \
  || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0))	  \
  || LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,18,0,0) \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0)   \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	return vmalloc_sync_mappings();
}

#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	return vmalloc_sync_all();
}

#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

#endif

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,12,0))
static inline
void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
{
	void *ret;

	ret = kvmalloc_node(size, flags, node);
	if (is_vmalloc_addr(ret)) {
		/*
		 * Make sure we don't trigger recursive page faults in the
		 * tracing fast path.
		 */
		wrapper_vmalloc_sync_mappings();
	}
	return ret;
}

static inline
void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
{
	return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node);
}

static inline
void *lttng_kvmalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void *lttng_kvzalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void lttng_kvfree(const void *addr)
{
	kvfree(addr);
}

#else

#include <linux/slab.h>

static inline
void print_vmalloc_node_range_warning(void)
{
	printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n");
	printk_once(KERN_WARNING "LTTng: Tracer performance will be degraded on NUMA systems.\n");
	printk_once(KERN_WARNING "LTTng: Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
}

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0))

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				vm_flags, node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/*
 * Canary function to check for '__vmalloc_node_range()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
 *                           unsigned long start, unsigned long end, gfp_t gfp_mask,
 *                           pgprot_t prot, unsigned long vm_flags, int node,
 *                           const void *caller);
 */
static inline
void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
			vm_flags, node, caller);
}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0))

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, const void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/*
 * Canary function to check for '__vmalloc_node_range()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
 *                           unsigned long start, unsigned long end, gfp_t gfp_mask,
 *                           pgprot_t prot, unsigned long vm_flags, int node,
 *                           const void *caller);
 */
static inline
void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, const void *caller)
{
	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
			node, caller);
}

#else /* (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0)) */

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/*
 * Canary function to check for '__vmalloc_node_range()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
 *                           unsigned long start, unsigned long end, gfp_t gfp_mask,
 *                           pgprot_t prot, unsigned long vm_flags, int node,
 *                           void *caller);
 */
static inline
void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, void *caller)
{
	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
			node, caller);
}

#endif

/**
 * lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
 * failure, fall back to non-contiguous (vmalloc) allocation.
 * @size: size of the request.
 * @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
 *
 * Uses kmalloc to get the memory but if the allocation fails then falls back
 * to the vmalloc allocator. Use lttng_kvfree to free the memory.
 *
 * Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
 */
static inline
void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
{
	void *ret;

	/*
	 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
	 * so the given set of flags has to be compatible.
	 */
	WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);

	/*
	 * If the allocation fits in a single page, do not fallback.
	 */
	if (size <= PAGE_SIZE) {
		return kmalloc_node(size, flags, node);
	}

	/*
	 * Make sure that larger requests are not too disruptive - no OOM
	 * killer and no allocation failure warnings as we have a fallback
	 */
	ret = kmalloc_node(size, flags | __GFP_NOWARN | __GFP_NORETRY, node);
	if (!ret) {
		ret = __lttng_vmalloc_node_range(size, 1,
				VMALLOC_START, VMALLOC_END,
				flags | __GFP_HIGHMEM, PAGE_KERNEL, 0,
				node, __builtin_return_address(0));
		/*
		 * Make sure we don't trigger recursive page faults in the
		 * tracing fast path.
		 */
		wrapper_vmalloc_sync_mappings();
	}
	return ret;
}

static inline
void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
{
	return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node);
}

static inline
void *lttng_kvmalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void *lttng_kvzalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void lttng_kvfree(const void *addr)
{
	if (is_vmalloc_addr(addr)) {
		vfree(addr);
	} else {
		kfree(addr);
	}
}
#endif

#endif /* _LTTNG_WRAPPER_VMALLOC_H */
Commit	Line	Data
b7cdc182	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
9f36eaed	2	*
886d51a3	3	* wrapper/vmalloc.h
6d2a620c MD	4	*
	5	* wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
	6	* available, else we need to have a kernel that exports this function to GPL
	7	* modules.
	8	*
886d51a3	9	* Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6d2a620c MD	10	*/
6d2a620c MD	11
9f36eaed MJ	12	#ifndef _LTTNG_WRAPPER_VMALLOC_H
	13	#define _LTTNG_WRAPPER_VMALLOC_H
	14
5f4c791e	15	#include <lttng/kernel-version.h>
48f5e0b5	16	#include <linux/vmalloc.h>
45fe4e1a	17	#include <linux/slab.h>
01ab5113	18	#include <linux/mm.h>
48f5e0b5	19
6d2a620c MD	20	#ifdef CONFIG_KALLSYMS
	21
	22	#include <linux/kallsyms.h>
5a2f5e92	23	#include <wrapper/kallsyms.h>
2b3dbafc	24	#include <lttng/kernel-version.h>
6d2a620c	25
5f4c791e	26	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))
0dcc94fa MJ	27
	28	/*
	29	* wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
	30	* are now synchronized when they are created or torn down.
	31	*/
	32	static inline
	33	void wrapper_vmalloc_sync_mappings(void)
	34	{}
	35
5f4c791e	36	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
9bfe744a	37	\|\| LTTNG_KERNEL_RANGE(5,5,12, 5,6,0) \
2b3dbafc OP	38	\|\| LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
	39	\|\| LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
	40	\|\| LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
	41	\|\| LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
	42	\|\| LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
05355f0b	43	\|\| LTTNG_KERNEL_RANGE(4,4,218, 4,5,0) \
2e4c781e SB	44	\|\| LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,16,0,0) \
2e4c781e SB	45	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0) \
05355f0b MJ	46	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0) \
05355f0b MJ	47	\|\| LTTNG_RHEL_KERNEL_RANGE(4,18,0,240,0,0, 4,19,0,0,0,0))
263b6c88 MD	48
	49	static inline
	50	void wrapper_vmalloc_sync_mappings(void)
	51	{
	52	void (*vmalloc_sync_mappings_sym)(void);
	53
	54	vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
	55	if (vmalloc_sync_mappings_sym) {
	56	vmalloc_sync_mappings_sym();
	57	} else {
	58	#ifdef CONFIG_X86
	59	/*
	60	* Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
	61	* trigger recursive page faults.
	62	*/
	63	printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
5a15f70c	64	printk_once(KERN_WARNING "LTTng: Page fault handler and NMI tracing might trigger faults.\n");
263b6c88 MD	65	#endif
	66	}
	67	}
	68
3dfec228 MJ	69	/*
	70	* Canary function to check for 'vmalloc_sync_mappings()' at compile time.
	71	*
	72	* From 'include/linux/vmalloc.h':
	73	*
	74	* void vmalloc_sync_mappings(void);
	75	*/
	76	static inline
	77	void __canary__vmalloc_sync_mappings(void)
	78	{
	79	vmalloc_sync_mappings();
	80	}
	81
5f4c791e	82	#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
263b6c88 MD	83
263b6c88 MD	84	/*
3dfec228	85	* Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.6.
263b6c88	86	*/
6d2a620c	87	static inline
263b6c88	88	void wrapper_vmalloc_sync_mappings(void)
6d2a620c MD	89	{
	90	void (*vmalloc_sync_all_sym)(void);
	91
c539a324	92	vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
6d2a620c MD	93	if (vmalloc_sync_all_sym) {
	94	vmalloc_sync_all_sym();
	95	} else {
	96	#ifdef CONFIG_X86
	97	/*
	98	* Only x86 needs vmalloc_sync_all to make sure LTTng does not
	99	* trigger recursive page faults.
	100	*/
e36de50d	101	printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n");
5a15f70c	102	printk_once(KERN_WARNING "LTTng: Page fault handler and NMI tracing might trigger faults.\n");
6d2a620c MD	103	#endif
	104	}
	105	}
263b6c88	106
3dfec228 MJ	107	/*
	108	* Canary function to check for 'vmalloc_sync_all()' at compile time.
	109	*
	110	* From 'include/linux/vmalloc.h':
	111	*
	112	* void vmalloc_sync_all(void);
	113	*/
	114	static inline
	115	void __canary__vmalloc_sync_all(void)
	116	{
	117	vmalloc_sync_all();
	118	}
	119
5f4c791e	120	#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
263b6c88	121
f740341a	122	#else /* CONFIG_KALLSYMS */
6d2a620c	123
5f4c791e	124	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))
f740341a MJ	125
	126	/*
	127	* wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
	128	* are now synchronized when they are created or torn down.
	129	*/
	130	static inline
	131	void wrapper_vmalloc_sync_mappings(void)
	132	{}
	133
5f4c791e	134	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
9bfe744a	135	\|\| LTTNG_KERNEL_RANGE(5,5,12, 5,6,0) \
2b3dbafc OP	136	\|\| LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
	137	\|\| LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
	138	\|\| LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
	139	\|\| LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
	140	\|\| LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
2e4c781e SB	141	\|\| LTTNG_KERNEL_RANGE(4,4,218, 4,5,0)) \
	142	\|\| LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,18,0,0) \
	143	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0) \
	144	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)
263b6c88	145
6d2a620c	146	static inline
263b6c88 MD	147	void wrapper_vmalloc_sync_mappings(void)
	148	{
	149	return vmalloc_sync_mappings();
	150	}
	151
5f4c791e	152	#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
263b6c88 MD	153
	154	static inline
	155	void wrapper_vmalloc_sync_mappings(void)
6d2a620c MD	156	{
	157	return vmalloc_sync_all();
	158	}
263b6c88	159
5f4c791e	160	#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
263b6c88	161
6d2a620c	162	#endif
b13f3ebe	163
5f4c791e	164	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,12,0))
48f5e0b5 MJ	165	static inline
	166	void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
	167	{
	168	void *ret;
	169
	170	ret = kvmalloc_node(size, flags, node);
	171	if (is_vmalloc_addr(ret)) {
	172	/*
	173	* Make sure we don't trigger recursive page faults in the
	174	* tracing fast path.
	175	*/
263b6c88	176	wrapper_vmalloc_sync_mappings();
48f5e0b5 MJ	177	}
	178	return ret;
	179	}
	180
	181	static inline
	182	void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
	183	{
	184	return lttng_kvmalloc_node(size, flags \| __GFP_ZERO, node);
	185	}
	186
	187	static inline
	188	void *lttng_kvmalloc(unsigned long size, gfp_t flags)
	189	{
	190	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
	191	}
	192
	193	static inline
	194	void *lttng_kvzalloc(unsigned long size, gfp_t flags)
	195	{
	196	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
	197	}
	198
	199	static inline
	200	void lttng_kvfree(const void *addr)
	201	{
	202	kvfree(addr);
	203	}
	204
	205	#else
	206
	207	#include <linux/slab.h>
48f5e0b5	208
20eb87c9 MD	209	static inline
	210	void print_vmalloc_node_range_warning(void)
	211	{
	212	printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n");
5a15f70c MJ	213	printk_once(KERN_WARNING "LTTng: Tracer performance will be degraded on NUMA systems.\n");
5a15f70c MJ	214	printk_once(KERN_WARNING "LTTng: Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
20eb87c9 MD	215	}
20eb87c9 MD	216
5f4c791e	217	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0))
fd094ddf	218
48f5e0b5 MJ	219	/*
	220	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	221	*/
	222	static inline
20eb87c9 MD	223	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	224	unsigned long start, unsigned long end, gfp_t gfp_mask,
	225	pgprot_t prot, unsigned long vm_flags, int node,
	226	const void *caller)
48f5e0b5	227	{
48f5e0b5 MJ	228	#ifdef CONFIG_KALLSYMS
48f5e0b5 MJ	229	/*
20eb87c9	230	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
48f5e0b5	231	*/
20eb87c9 MD	232	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	233	unsigned long start, unsigned long end, gfp_t gfp_mask,
	234	pgprot_t prot, unsigned long vm_flags, int node,
	235	const void *caller);
	236
	237	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	238	if (lttng__vmalloc_node_range)
	239	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	240	vm_flags, node, caller);
48f5e0b5	241	#endif
20eb87c9 MD	242	if (node != NUMA_NO_NODE)
	243	print_vmalloc_node_range_warning();
	244	return __vmalloc(size, gfp_mask, prot);
48f5e0b5	245	}
3dfec228 MJ	246
	247	/*
	248	* Canary function to check for '__vmalloc_node_range()' at compile time.
	249	*
	250	* From 'include/linux/vmalloc.h':
	251	*
	252	* extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
	253	* unsigned long start, unsigned long end, gfp_t gfp_mask,
	254	* pgprot_t prot, unsigned long vm_flags, int node,
	255	* const void *caller);
	256	*/
	257	static inline
	258	void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	259	unsigned long start, unsigned long end, gfp_t gfp_mask,
	260	pgprot_t prot, unsigned long vm_flags, int node,
	261	const void *caller)
	262	{
	263	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	264	vm_flags, node, caller);
	265	}
48f5e0b5	266
5f4c791e	267	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0))
fd094ddf MJ	268
	269	/*
	270	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	271	*/
	272	static inline
	273	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	274	unsigned long start, unsigned long end, gfp_t gfp_mask,
	275	pgprot_t prot, unsigned long vm_flags, int node,
	276	const void *caller)
	277	{
	278	#ifdef CONFIG_KALLSYMS
	279	/*
	280	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	281	*/
	282	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	283	unsigned long start, unsigned long end, gfp_t gfp_mask,
	284	pgprot_t prot, int node, const void *caller);
	285
	286	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	287	if (lttng__vmalloc_node_range)
	288	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	289	node, caller);
	290	#endif
	291	if (node != NUMA_NO_NODE)
	292	print_vmalloc_node_range_warning();
	293	return __vmalloc(size, gfp_mask, prot);
	294	}
	295
	296	/*
	297	* Canary function to check for '__vmalloc_node_range()' at compile time.
	298	*
	299	* From 'include/linux/vmalloc.h':
	300	*
	301	* extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
	302	* unsigned long start, unsigned long end, gfp_t gfp_mask,
	303	* pgprot_t prot, unsigned long vm_flags, int node,
	304	* const void *caller);
	305	*/
	306	static inline
	307	void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	308	unsigned long start, unsigned long end, gfp_t gfp_mask,
	309	pgprot_t prot, int node, const void *caller)
	310	{
	311	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	312	node, caller);
	313	}
	314
5f4c791e	315	#else /* (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0)) */
0c8a119a MJ	316
	317	/*
	318	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	319	*/
	320	static inline
	321	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	322	unsigned long start, unsigned long end, gfp_t gfp_mask,
	323	pgprot_t prot, unsigned long vm_flags, int node,
	324	void *caller)
	325	{
	326	#ifdef CONFIG_KALLSYMS
	327	/*
	328	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	329	*/
	330	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	331	unsigned long start, unsigned long end, gfp_t gfp_mask,
	332	pgprot_t prot, int node, void *caller);
	333
	334	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	335	if (lttng__vmalloc_node_range)
	336	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	337	node, caller);
	338	#endif
	339	if (node != NUMA_NO_NODE)
	340	print_vmalloc_node_range_warning();
	341	return __vmalloc(size, gfp_mask, prot);
	342	}
	343
	344	/*
	345	* Canary function to check for '__vmalloc_node_range()' at compile time.
	346	*
	347	* From 'include/linux/vmalloc.h':
	348	*
	349	* extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
	350	* unsigned long start, unsigned long end, gfp_t gfp_mask,
	351	* pgprot_t prot, unsigned long vm_flags, int node,
	352	* void *caller);
	353	*/
	354	static inline
	355	void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	356	unsigned long start, unsigned long end, gfp_t gfp_mask,
	357	pgprot_t prot, int node, void *caller)
	358	{
	359	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	360	node, caller);
	361	}
	362
fd094ddf MJ	363	#endif
fd094ddf MJ	364
48f5e0b5 MJ	365	/**
	366	* lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
	367	* failure, fall back to non-contiguous (vmalloc) allocation.
	368	* @size: size of the request.
	369	* @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
	370	*
	371	* Uses kmalloc to get the memory but if the allocation fails then falls back
	372	* to the vmalloc allocator. Use lttng_kvfree to free the memory.
	373	*
	374	* Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
	375	*/
	376	static inline
	377	void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
	378	{
	379	void *ret;
	380
	381	/*
	382	* vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
	383	* so the given set of flags has to be compatible.
	384	*/
	385	WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);
	386
	387	/*
	388	* If the allocation fits in a single page, do not fallback.
	389	*/
	390	if (size <= PAGE_SIZE) {
	391	return kmalloc_node(size, flags, node);
	392	}
	393
	394	/*
	395	* Make sure that larger requests are not too disruptive - no OOM
	396	* killer and no allocation failure warnings as we have a fallback
	397	*/
	398	ret = kmalloc_node(size, flags \| __GFP_NOWARN \| __GFP_NORETRY, node);
	399	if (!ret) {
20eb87c9 MD	400	ret = __lttng_vmalloc_node_range(size, 1,
	401	VMALLOC_START, VMALLOC_END,
	402	flags \| __GFP_HIGHMEM, PAGE_KERNEL, 0,
	403	node, __builtin_return_address(0));
48f5e0b5 MJ	404	/*
	405	* Make sure we don't trigger recursive page faults in the
	406	* tracing fast path.
	407	*/
f3e4ba5d	408	wrapper_vmalloc_sync_mappings();
48f5e0b5 MJ	409	}
	410	return ret;
	411	}
	412
	413	static inline
	414	void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
	415	{
	416	return lttng_kvmalloc_node(size, flags \| __GFP_ZERO, node);
	417	}
	418
	419	static inline
	420	void *lttng_kvmalloc(unsigned long size, gfp_t flags)
	421	{
	422	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
	423	}
	424
	425	static inline
	426	void *lttng_kvzalloc(unsigned long size, gfp_t flags)
	427	{
	428	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
	429	}
	430
	431	static inline
	432	void lttng_kvfree(const void *addr)
	433	{
	434	if (is_vmalloc_addr(addr)) {
	435	vfree(addr);
	436	} else {
	437	kfree(addr);
	438	}
	439	}
	440	#endif
	441
a90917c3	442	#endif /* _LTTNG_WRAPPER_VMALLOC_H */