[lttng-ust.git] / liblttng-ust-libc-wrapper / lttng-ust-malloc.c

/*
 * Copyright (C) 2009  Pierre-Marc Fournier
 * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 */

#define _GNU_SOURCE
#include <dlfcn.h>
#include <sys/types.h>
#include <stdio.h>
#include <assert.h>
#include <urcu/system.h>
#include <urcu/uatomic.h>
#include <urcu/compiler.h>
#include <lttng/align.h>

#define TRACEPOINT_DEFINE
#define TRACEPOINT_CREATE_PROBES
#include "ust_libc.h"

#define STATIC_CALLOC_LEN 4096
static char static_calloc_buf[STATIC_CALLOC_LEN];
static unsigned long static_calloc_buf_offset;

struct alloc_functions {
	void *(*calloc)(size_t nmemb, size_t size);
	void *(*malloc)(size_t size);
	void (*free)(void *ptr);
	void *(*realloc)(void *ptr, size_t size);
	void *(*memalign)(size_t alignment, size_t size);
	int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
};

static
struct alloc_functions cur_alloc;

/*
 * Static allocator to use when initially executing dlsym(). It keeps a
 * size_t value of each object size prior to the object.
 */
static
void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
{
	size_t prev_offset, new_offset, res_offset, aligned_offset;

	if (nmemb * size == 0) {
		return NULL;
	}

	/*
	 * Protect static_calloc_buf_offset from concurrent updates
	 * using a cmpxchg loop rather than a mutex to remove a
	 * dependency on pthread. This will minimize the risk of bad
	 * interaction between mutex and malloc instrumentation.
	 */
	res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
	do {
		prev_offset = res_offset;
		aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
		new_offset = aligned_offset + nmemb * size;
		if (new_offset > sizeof(static_calloc_buf)) {
			abort();
		}
	} while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
			prev_offset, new_offset)) != prev_offset);
	*(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
	return &static_calloc_buf[aligned_offset];
}

static
void *static_calloc(size_t nmemb, size_t size)
{
	void *retval;

	retval = static_calloc_aligned(nmemb, size, 1);
	tracepoint(ust_libc, calloc, nmemb, size, retval);
	return retval;
}

static
void *static_malloc(size_t size)
{
	void *retval;

	retval = static_calloc_aligned(1, size, 1);
	tracepoint(ust_libc, malloc, size, retval);
	return retval;
}

static
void static_free(void *ptr)
{
	/* no-op. */
	tracepoint(ust_libc, free, ptr);
}

static
void *static_realloc(void *ptr, size_t size)
{
	size_t *old_size = NULL;
	void *retval;

	if (size == 0) {
		retval = NULL;
		goto end;
	}

	if (ptr) {
		old_size = (size_t *) ptr - 1;
		if (size <= *old_size) {
			/* We can re-use the old entry. */
			*old_size = size;
			retval = ptr;
			goto end;
		}
	}
	/* We need to expand. Don't free previous memory location. */
	retval = static_calloc_aligned(1, size, 1);
	assert(retval);
	if (ptr)
		memcpy(retval, ptr, *old_size);
end:
	tracepoint(ust_libc, realloc, ptr, size, retval);
	return retval;
}

static
void *static_memalign(size_t alignment, size_t size)
{
	void *retval;

	retval = static_calloc_aligned(1, size, alignment);
	tracepoint(ust_libc, memalign, alignment, size, retval);
	return retval;
}

static
int static_posix_memalign(void **memptr, size_t alignment, size_t size)
{
	int retval = 0;
	void *ptr;

	/* Check for power of 2, larger than void *. */
	if (alignment & (alignment - 1)
			|| alignment < sizeof(void *)
			|| alignment == 0) {
		retval = EINVAL;
		goto end;
	}
	ptr = static_calloc_aligned(1, size, alignment);
	*memptr = ptr;
	if (size && !ptr)
		retval = ENOMEM;
end:
	tracepoint(ust_libc, posix_memalign, *memptr, alignment, size, retval);
	return 0;
}

static
void setup_static_allocator(void)
{
	assert(cur_alloc.calloc == NULL);
	cur_alloc.calloc = static_calloc;
	assert(cur_alloc.malloc == NULL);
	cur_alloc.malloc = static_malloc;
	assert(cur_alloc.free == NULL);
	cur_alloc.free = static_free;
	assert(cur_alloc.realloc == NULL);
	cur_alloc.realloc = static_realloc;
	assert(cur_alloc.memalign == NULL);
	cur_alloc.memalign = static_memalign;
	assert(cur_alloc.posix_memalign == NULL);
	cur_alloc.posix_memalign = static_posix_memalign;
}

static
void lookup_all_symbols(void)
{
	struct alloc_functions af;

	/*
	 * Temporarily redirect allocation functions to
	 * static_calloc_aligned, and free function to static_free
	 * (no-op), until the dlsym lookup has completed.
	 */
	setup_static_allocator();

	/* Perform the actual lookups */
	af.calloc = dlsym(RTLD_NEXT, "calloc");
	af.malloc = dlsym(RTLD_NEXT, "malloc");
	af.free = dlsym(RTLD_NEXT, "free");
	af.realloc = dlsym(RTLD_NEXT, "realloc");
	af.memalign = dlsym(RTLD_NEXT, "memalign");
	af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");

	/* Populate the new allocator functions */
	memcpy(&cur_alloc, &af, sizeof(cur_alloc));
}

void *malloc(size_t size)
{
	void *retval;

	if (cur_alloc.malloc == NULL) {
		lookup_all_symbols();
		if (cur_alloc.malloc == NULL) {
			fprintf(stderr, "mallocwrap: unable to find malloc\n");
			abort();
		}
	}
	retval = cur_alloc.malloc(size);
	tracepoint(ust_libc, malloc, size, retval);
	return retval;
}

void free(void *ptr)
{
	tracepoint(ust_libc, free, ptr);

	/*
	 * Check whether the memory was allocated with
	 * static_calloc_align, in which case there is nothing to free.
	 */
	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
			(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
		return;
	}

	if (cur_alloc.free == NULL) {
		lookup_all_symbols();
		if (cur_alloc.free == NULL) {
			fprintf(stderr, "mallocwrap: unable to find free\n");
			abort();
		}
	}
	cur_alloc.free(ptr);
}

void *calloc(size_t nmemb, size_t size)
{
	void *retval;

	if (cur_alloc.calloc == NULL) {
		lookup_all_symbols();
		if (cur_alloc.calloc == NULL) {
			fprintf(stderr, "callocwrap: unable to find calloc\n");
			abort();
		}
	}
	retval = cur_alloc.calloc(nmemb, size);
	tracepoint(ust_libc, calloc, nmemb, size, retval);
	return retval;
}

void *realloc(void *ptr, size_t size)
{
	void *retval;

	/* Check whether the memory was allocated with
	 * static_calloc_align, in which case there is nothing
	 * to free, and we need to copy the old data.
	 */
	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
			(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
		size_t *old_size;

		old_size = (size_t *) ptr - 1;
		if (cur_alloc.calloc == NULL) {
			lookup_all_symbols();
			if (cur_alloc.calloc == NULL) {
				fprintf(stderr, "reallocwrap: unable to find calloc\n");
				abort();
			}
		}
		retval = cur_alloc.calloc(1, size);
		if (retval) {
			memcpy(retval, ptr, *old_size);
		}
		goto end;
	}

	if (cur_alloc.realloc == NULL) {
		lookup_all_symbols();
		if (cur_alloc.realloc == NULL) {
			fprintf(stderr, "reallocwrap: unable to find realloc\n");
			abort();
		}
	}
	retval = cur_alloc.realloc(ptr, size);
end:
	tracepoint(ust_libc, realloc, ptr, size, retval);
	return retval;
}

void *memalign(size_t alignment, size_t size)
{
	void *retval;

	if (cur_alloc.memalign == NULL) {
		lookup_all_symbols();
		if (cur_alloc.memalign == NULL) {
			fprintf(stderr, "memalignwrap: unable to find memalign\n");
			abort();
		}
	}
	retval = cur_alloc.memalign(alignment, size);
	tracepoint(ust_libc, memalign, alignment, size, retval);
	return retval;
}

int posix_memalign(void **memptr, size_t alignment, size_t size)
{
	int retval;

	if (cur_alloc.posix_memalign == NULL) {
		lookup_all_symbols();
		if (cur_alloc.posix_memalign == NULL) {
			fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
			abort();
		}
	}
	retval = cur_alloc.posix_memalign(memptr, alignment, size);
	tracepoint(ust_libc, posix_memalign, *memptr, alignment, size, retval);
	return retval;
}

__attribute__((constructor))
void lttng_ust_malloc_wrapper_init(void)
{
	/* Initialization already done */
	if (cur_alloc.calloc) {
		return;
	}
	/*
	 * Ensure the allocator is in place before the process becomes
	 * multithreaded.
	 */
	lookup_all_symbols();
}
Commit	Line	Data
b27f8e75 MD	1	/*
b27f8e75 MD	2	* Copyright (C) 2009 Pierre-Marc Fournier
1622ba22	3	* Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
c39c72ee PMF	4	*
	5	* This library is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU Lesser General Public
	7	* License as published by the Free Software Foundation; either
	8	* version 2.1 of the License, or (at your option) any later version.
	9	*
	10	* This library is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* Lesser General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU Lesser General Public
	16	* License along with this library; if not, write to the Free Software
	17	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	18	*/
	19
e541a28d PMF	20	#define _GNU_SOURCE
	21	#include <dlfcn.h>
	22	#include <sys/types.h>
	23	#include <stdio.h>
2594a5b4	24	#include <assert.h>
4c3536e0 MD	25	#include <urcu/system.h>
4c3536e0 MD	26	#include <urcu/uatomic.h>
2594a5b4 MD	27	#include <urcu/compiler.h>
2594a5b4 MD	28	#include <lttng/align.h>
1622ba22 MD	29
	30	#define TRACEPOINT_DEFINE
	31	#define TRACEPOINT_CREATE_PROBES
	32	#include "ust_libc.h"
fbd8191b	33
f95b2888 SS	34	#define STATIC_CALLOC_LEN 4096
f95b2888 SS	35	static char static_calloc_buf[STATIC_CALLOC_LEN];
4c3536e0	36	static unsigned long static_calloc_buf_offset;
f95b2888	37
2594a5b4 MD	38	struct alloc_functions {
	39	void (calloc)(size_t nmemb, size_t size);
	40	void (malloc)(size_t size);
	41	void (free)(void ptr);
	42	void (realloc)(void *ptr, size_t size);
	43	void (memalign)(size_t alignment, size_t size);
	44	int (posix_memalign)(void *memptr, size_t alignment, size_t size);
	45	};
	46
	47	static
	48	struct alloc_functions cur_alloc;
	49
	50	/*
	51	* Static allocator to use when initially executing dlsym(). It keeps a
	52	* size_t value of each object size prior to the object.
	53	*/
	54	static
	55	void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
f95b2888	56	{
2594a5b4 MD	57	size_t prev_offset, new_offset, res_offset, aligned_offset;
	58
	59	if (nmemb * size == 0) {
	60	return NULL;
	61	}
f95b2888	62
4c3536e0 MD	63	/*
	64	* Protect static_calloc_buf_offset from concurrent updates
	65	* using a cmpxchg loop rather than a mutex to remove a
	66	* dependency on pthread. This will minimize the risk of bad
	67	* interaction between mutex and malloc instrumentation.
	68	*/
	69	res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
	70	do {
	71	prev_offset = res_offset;
2594a5b4 MD	72	aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
	73	new_offset = aligned_offset + nmemb * size;
	74	if (new_offset > sizeof(static_calloc_buf)) {
	75	abort();
4c3536e0	76	}
4c3536e0 MD	77	} while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
4c3536e0 MD	78	prev_offset, new_offset)) != prev_offset);
2594a5b4 MD	79	(size_t ) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
	80	return &static_calloc_buf[aligned_offset];
	81	}
	82
	83	static
	84	void *static_calloc(size_t nmemb, size_t size)
	85	{
	86	void *retval;
	87
	88	retval = static_calloc_aligned(nmemb, size, 1);
	89	tracepoint(ust_libc, calloc, nmemb, size, retval);
	90	return retval;
	91	}
	92
	93	static
	94	void *static_malloc(size_t size)
	95	{
	96	void *retval;
	97
	98	retval = static_calloc_aligned(1, size, 1);
	99	tracepoint(ust_libc, malloc, size, retval);
	100	return retval;
	101	}
	102
	103	static
	104	void static_free(void *ptr)
	105	{
	106	/* no-op. */
	107	tracepoint(ust_libc, free, ptr);
	108	}
	109
	110	static
	111	void static_realloc(void ptr, size_t size)
	112	{
	113	size_t *old_size = NULL;
	114	void *retval;
	115
	116	if (size == 0) {
	117	retval = NULL;
	118	goto end;
	119	}
	120
	121	if (ptr) {
	122	old_size = (size_t *) ptr - 1;
	123	if (size <= *old_size) {
	124	/* We can re-use the old entry. */
	125	*old_size = size;
	126	retval = ptr;
	127	goto end;
	128	}
	129	}
	130	/* We need to expand. Don't free previous memory location. */
	131	retval = static_calloc_aligned(1, size, 1);
	132	assert(retval);
	133	if (ptr)
	134	memcpy(retval, ptr, *old_size);
	135	end:
	136	tracepoint(ust_libc, realloc, ptr, size, retval);
	137	return retval;
	138	}
	139
	140	static
	141	void *static_memalign(size_t alignment, size_t size)
	142	{
143	void *retval;
144
145	retval = static_calloc_aligned(1, size, alignment);
146	tracepoint(ust_libc, memalign, alignment, size, retval);
147	return retval;
148	}
149
150	static
151	int static_posix_memalign(void **memptr, size_t alignment, size_t size)
152	{
153	int retval = 0;
154	void *ptr;
155
156	/* Check for power of 2, larger than void . /
157	if (alignment & (alignment - 1)
158	\|\| alignment < sizeof(void *)
159	\|\| alignment == 0) {
160	retval = EINVAL;
161	goto end;
162	}
163	ptr = static_calloc_aligned(1, size, alignment);
164	*memptr = ptr;
165	if (size && !ptr)
166	retval = ENOMEM;
167	end:
168	tracepoint(ust_libc, posix_memalign, *memptr, alignment, size, retval);
169	return 0;
170	}
171
172	static
173	void setup_static_allocator(void)
174	{
175	assert(cur_alloc.calloc == NULL);
176	cur_alloc.calloc = static_calloc;
177	assert(cur_alloc.malloc == NULL);
178	cur_alloc.malloc = static_malloc;
179	assert(cur_alloc.free == NULL);
180	cur_alloc.free = static_free;
181	assert(cur_alloc.realloc == NULL);
182	cur_alloc.realloc = static_realloc;
183	assert(cur_alloc.memalign == NULL);
184	cur_alloc.memalign = static_memalign;
185	assert(cur_alloc.posix_memalign == NULL);
186	cur_alloc.posix_memalign = static_posix_memalign;
187	}
188
189	static
190	void lookup_all_symbols(void)
191	{
192	struct alloc_functions af;
193
194	/*
195	* Temporarily redirect allocation functions to
196	* static_calloc_aligned, and free function to static_free
197	* (no-op), until the dlsym lookup has completed.
198	*/
199	setup_static_allocator();
200
201	/* Perform the actual lookups */
202	af.calloc = dlsym(RTLD_NEXT, "calloc");
203	af.malloc = dlsym(RTLD_NEXT, "malloc");
204	af.free = dlsym(RTLD_NEXT, "free");
205	af.realloc = dlsym(RTLD_NEXT, "realloc");
206	af.memalign = dlsym(RTLD_NEXT, "memalign");
207	af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
208
209	/* Populate the new allocator functions */
210	memcpy(&cur_alloc, &af, sizeof(cur_alloc));
f95b2888 SS	211	}
f95b2888 SS	212
e541a28d PMF	213	void *malloc(size_t size)
e541a28d PMF	214	{
1c184644 PMF	215	void *retval;
1c184644 PMF	216
2594a5b4 MD	217	if (cur_alloc.malloc == NULL) {
	218	lookup_all_symbols();
	219	if (cur_alloc.malloc == NULL) {
e541a28d	220	fprintf(stderr, "mallocwrap: unable to find malloc\n");
2594a5b4	221	abort();
e541a28d PMF	222	}
e541a28d PMF	223	}
2594a5b4	224	retval = cur_alloc.malloc(size);
1622ba22	225	tracepoint(ust_libc, malloc, size, retval);
1c184644 PMF	226	return retval;
	227	}
	228
	229	void free(void *ptr)
	230	{
2594a5b4	231	tracepoint(ust_libc, free, ptr);
f95b2888	232
2594a5b4 MD	233	/*
	234	* Check whether the memory was allocated with
	235	* static_calloc_align, in which case there is nothing to free.
f95b2888	236	*/
2594a5b4 MD	237	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
2594a5b4 MD	238	(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
f95b2888 SS	239	return;
f95b2888 SS	240	}
1c184644	241
2594a5b4 MD	242	if (cur_alloc.free == NULL) {
	243	lookup_all_symbols();
	244	if (cur_alloc.free == NULL) {
1c184644	245	fprintf(stderr, "mallocwrap: unable to find free\n");
2594a5b4	246	abort();
1c184644 PMF	247	}
1c184644 PMF	248	}
2594a5b4	249	cur_alloc.free(ptr);
e541a28d	250	}
f95b2888 SS	251
	252	void *calloc(size_t nmemb, size_t size)
	253	{
f95b2888 SS	254	void *retval;
f95b2888 SS	255
2594a5b4 MD	256	if (cur_alloc.calloc == NULL) {
	257	lookup_all_symbols();
	258	if (cur_alloc.calloc == NULL) {
f95b2888	259	fprintf(stderr, "callocwrap: unable to find calloc\n");
2594a5b4	260	abort();
f95b2888 SS	261	}
f95b2888 SS	262	}
2594a5b4	263	retval = cur_alloc.calloc(nmemb, size);
f95b2888 SS	264	tracepoint(ust_libc, calloc, nmemb, size, retval);
	265	return retval;
	266	}
	267
	268	void realloc(void ptr, size_t size)
	269	{
f95b2888 SS	270	void *retval;
f95b2888 SS	271
2594a5b4 MD	272	/* Check whether the memory was allocated with
	273	* static_calloc_align, in which case there is nothing
	274	* to free, and we need to copy the old data.
	275	*/
	276	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
	277	(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
	278	size_t *old_size;
	279
	280	old_size = (size_t *) ptr - 1;
	281	if (cur_alloc.calloc == NULL) {
	282	lookup_all_symbols();
	283	if (cur_alloc.calloc == NULL) {
	284	fprintf(stderr, "reallocwrap: unable to find calloc\n");
	285	abort();
	286	}
	287	}
	288	retval = cur_alloc.calloc(1, size);
	289	if (retval) {
	290	memcpy(retval, ptr, *old_size);
	291	}
	292	goto end;
	293	}
	294
	295	if (cur_alloc.realloc == NULL) {
	296	lookup_all_symbols();
	297	if (cur_alloc.realloc == NULL) {
f95b2888	298	fprintf(stderr, "reallocwrap: unable to find realloc\n");
2594a5b4	299	abort();
f95b2888 SS	300	}
f95b2888 SS	301	}
2594a5b4 MD	302	retval = cur_alloc.realloc(ptr, size);
2594a5b4 MD	303	end:
f95b2888 SS	304	tracepoint(ust_libc, realloc, ptr, size, retval);
	305	return retval;
	306	}
9d34b226 SS	307
	308	void *memalign(size_t alignment, size_t size)
	309	{
9d34b226 SS	310	void *retval;
9d34b226 SS	311
2594a5b4 MD	312	if (cur_alloc.memalign == NULL) {
	313	lookup_all_symbols();
	314	if (cur_alloc.memalign == NULL) {
9d34b226	315	fprintf(stderr, "memalignwrap: unable to find memalign\n");
2594a5b4	316	abort();
9d34b226 SS	317	}
9d34b226 SS	318	}
2594a5b4	319	retval = cur_alloc.memalign(alignment, size);
9d34b226 SS	320	tracepoint(ust_libc, memalign, alignment, size, retval);
	321	return retval;
	322	}
	323
	324	int posix_memalign(void **memptr, size_t alignment, size_t size)
	325	{
9d34b226 SS	326	int retval;
9d34b226 SS	327
2594a5b4 MD	328	if (cur_alloc.posix_memalign == NULL) {
	329	lookup_all_symbols();
	330	if (cur_alloc.posix_memalign == NULL) {
9d34b226	331	fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
2594a5b4	332	abort();
9d34b226 SS	333	}
9d34b226 SS	334	}
2594a5b4	335	retval = cur_alloc.posix_memalign(memptr, alignment, size);
9d34b226 SS	336	tracepoint(ust_libc, posix_memalign, *memptr, alignment, size, retval);
	337	return retval;
	338	}
2594a5b4 MD	339
	340	__attribute__((constructor))
	341	void lttng_ust_malloc_wrapper_init(void)
	342	{
	343	/* Initialization already done */
	344	if (cur_alloc.calloc) {
	345	return;
	346	}
	347	/*
	348	* Ensure the allocator is in place before the process becomes
	349	* multithreaded.
	350	*/
	351	lookup_all_symbols();
	352	}