2 * SPDX-License-Identifier: LGPL-2.1-or-later
4 * Copyright (C) 2009 Pierre-Marc Fournier
5 * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9 * Do _not_ define _LGPL_SOURCE because we don't want to create a
10 * circular dependency loop between this malloc wrapper, liburcu and
13 #include <ust-dlfcn.h>
14 #include <sys/types.h>
17 #include <urcu/system.h>
18 #include <urcu/uatomic.h>
19 #include <urcu/compiler.h>
20 #include <urcu/tls-compat.h>
21 #include <urcu/arch.h>
22 #include <ust-helper.h>
23 #include "ust-compat.h"
25 #define TRACEPOINT_DEFINE
26 #define TRACEPOINT_CREATE_PROBES
27 #define TP_IP_PARAM ip
30 #define STATIC_CALLOC_LEN 4096
31 static char static_calloc_buf
[STATIC_CALLOC_LEN
];
32 static unsigned long static_calloc_buf_offset
;
34 struct alloc_functions
{
35 void *(*calloc
)(size_t nmemb
, size_t size
);
36 void *(*malloc
)(size_t size
);
37 void (*free
)(void *ptr
);
38 void *(*realloc
)(void *ptr
, size_t size
);
39 void *(*memalign
)(size_t alignment
, size_t size
);
40 int (*posix_memalign
)(void **memptr
, size_t alignment
, size_t size
);
44 struct alloc_functions cur_alloc
;
47 * Make sure our own use of the LTS compat layer will not cause infinite
48 * recursion by calling calloc.
52 void *static_calloc(size_t nmemb
, size_t size
);
55 * pthread mutex replacement for URCU tls compat layer.
57 static int ust_malloc_lock
;
59 static __attribute__((unused
))
60 void ust_malloc_spin_lock(pthread_mutex_t
*lock
)
63 * The memory barrier within cmpxchg takes care of ordering
64 * memory accesses with respect to the start of the critical
67 while (uatomic_cmpxchg(&ust_malloc_lock
, 0, 1) != 0)
71 static __attribute__((unused
))
72 void ust_malloc_spin_unlock(pthread_mutex_t
*lock
)
75 * Ensure memory accesses within the critical section do not
79 uatomic_set(&ust_malloc_lock
, 0);
82 #define calloc static_calloc
83 #define pthread_mutex_lock ust_malloc_spin_lock
84 #define pthread_mutex_unlock ust_malloc_spin_unlock
85 static DEFINE_URCU_TLS(int, malloc_nesting
);
86 #undef pthread_mutex_unlock
87 #undef pthread_mutex_lock
91 * Static allocator to use when initially executing dlsym(). It keeps a
92 * size_t value of each object size prior to the object.
95 void *static_calloc_aligned(size_t nmemb
, size_t size
, size_t alignment
)
97 size_t prev_offset
, new_offset
, res_offset
, aligned_offset
;
99 if (nmemb
* size
== 0) {
104 * Protect static_calloc_buf_offset from concurrent updates
105 * using a cmpxchg loop rather than a mutex to remove a
106 * dependency on pthread. This will minimize the risk of bad
107 * interaction between mutex and malloc instrumentation.
109 res_offset
= CMM_LOAD_SHARED(static_calloc_buf_offset
);
111 prev_offset
= res_offset
;
112 aligned_offset
= LTTNG_UST_ALIGN(prev_offset
+ sizeof(size_t), alignment
);
113 new_offset
= aligned_offset
+ nmemb
* size
;
114 if (new_offset
> sizeof(static_calloc_buf
)) {
117 } while ((res_offset
= uatomic_cmpxchg(&static_calloc_buf_offset
,
118 prev_offset
, new_offset
)) != prev_offset
);
119 *(size_t *) &static_calloc_buf
[aligned_offset
- sizeof(size_t)] = size
;
120 return &static_calloc_buf
[aligned_offset
];
124 void *static_calloc(size_t nmemb
, size_t size
)
128 retval
= static_calloc_aligned(nmemb
, size
, 1);
133 void *static_malloc(size_t size
)
137 retval
= static_calloc_aligned(1, size
, 1);
142 void static_free(void *ptr
)
148 void *static_realloc(void *ptr
, size_t size
)
150 size_t *old_size
= NULL
;
159 old_size
= (size_t *) ptr
- 1;
160 if (size
<= *old_size
) {
161 /* We can re-use the old entry. */
167 /* We need to expand. Don't free previous memory location. */
168 retval
= static_calloc_aligned(1, size
, 1);
171 memcpy(retval
, ptr
, *old_size
);
177 void *static_memalign(size_t alignment
, size_t size
)
181 retval
= static_calloc_aligned(1, size
, alignment
);
186 int static_posix_memalign(void **memptr
, size_t alignment
, size_t size
)
190 /* Check for power of 2, larger than void *. */
191 if (alignment
& (alignment
- 1)
192 || alignment
< sizeof(void *)
196 ptr
= static_calloc_aligned(1, size
, alignment
);
203 void setup_static_allocator(void)
205 assert(cur_alloc
.calloc
== NULL
);
206 cur_alloc
.calloc
= static_calloc
;
207 assert(cur_alloc
.malloc
== NULL
);
208 cur_alloc
.malloc
= static_malloc
;
209 assert(cur_alloc
.free
== NULL
);
210 cur_alloc
.free
= static_free
;
211 assert(cur_alloc
.realloc
== NULL
);
212 cur_alloc
.realloc
= static_realloc
;
213 assert(cur_alloc
.memalign
== NULL
);
214 cur_alloc
.memalign
= static_memalign
;
215 assert(cur_alloc
.posix_memalign
== NULL
);
216 cur_alloc
.posix_memalign
= static_posix_memalign
;
220 void lookup_all_symbols(void)
222 struct alloc_functions af
;
225 * Temporarily redirect allocation functions to
226 * static_calloc_aligned, and free function to static_free
227 * (no-op), until the dlsym lookup has completed.
229 setup_static_allocator();
231 /* Perform the actual lookups */
232 af
.calloc
= dlsym(RTLD_NEXT
, "calloc");
233 af
.malloc
= dlsym(RTLD_NEXT
, "malloc");
234 af
.free
= dlsym(RTLD_NEXT
, "free");
235 af
.realloc
= dlsym(RTLD_NEXT
, "realloc");
236 af
.memalign
= dlsym(RTLD_NEXT
, "memalign");
237 af
.posix_memalign
= dlsym(RTLD_NEXT
, "posix_memalign");
239 /* Populate the new allocator functions */
240 memcpy(&cur_alloc
, &af
, sizeof(cur_alloc
));
243 void *malloc(size_t size
)
247 URCU_TLS(malloc_nesting
)++;
248 if (cur_alloc
.malloc
== NULL
) {
249 lookup_all_symbols();
250 if (cur_alloc
.malloc
== NULL
) {
251 fprintf(stderr
, "mallocwrap: unable to find malloc\n");
255 retval
= cur_alloc
.malloc(size
);
256 if (URCU_TLS(malloc_nesting
) == 1) {
257 tracepoint(lttng_ust_libc
, malloc
,
258 size
, retval
, LTTNG_UST_CALLER_IP());
260 URCU_TLS(malloc_nesting
)--;
266 URCU_TLS(malloc_nesting
)++;
268 * Check whether the memory was allocated with
269 * static_calloc_align, in which case there is nothing to free.
271 if (caa_unlikely((char *)ptr
>= static_calloc_buf
&&
272 (char *)ptr
< static_calloc_buf
+ STATIC_CALLOC_LEN
)) {
276 if (URCU_TLS(malloc_nesting
) == 1) {
277 tracepoint(lttng_ust_libc
, free
,
278 ptr
, LTTNG_UST_CALLER_IP());
281 if (cur_alloc
.free
== NULL
) {
282 lookup_all_symbols();
283 if (cur_alloc
.free
== NULL
) {
284 fprintf(stderr
, "mallocwrap: unable to find free\n");
290 URCU_TLS(malloc_nesting
)--;
293 void *calloc(size_t nmemb
, size_t size
)
297 URCU_TLS(malloc_nesting
)++;
298 if (cur_alloc
.calloc
== NULL
) {
299 lookup_all_symbols();
300 if (cur_alloc
.calloc
== NULL
) {
301 fprintf(stderr
, "callocwrap: unable to find calloc\n");
305 retval
= cur_alloc
.calloc(nmemb
, size
);
306 if (URCU_TLS(malloc_nesting
) == 1) {
307 tracepoint(lttng_ust_libc
, calloc
,
308 nmemb
, size
, retval
, LTTNG_UST_CALLER_IP());
310 URCU_TLS(malloc_nesting
)--;
314 void *realloc(void *ptr
, size_t size
)
318 URCU_TLS(malloc_nesting
)++;
320 * Check whether the memory was allocated with
321 * static_calloc_align, in which case there is nothing
322 * to free, and we need to copy the old data.
324 if (caa_unlikely((char *)ptr
>= static_calloc_buf
&&
325 (char *)ptr
< static_calloc_buf
+ STATIC_CALLOC_LEN
)) {
328 old_size
= (size_t *) ptr
- 1;
329 if (cur_alloc
.calloc
== NULL
) {
330 lookup_all_symbols();
331 if (cur_alloc
.calloc
== NULL
) {
332 fprintf(stderr
, "reallocwrap: unable to find calloc\n");
336 retval
= cur_alloc
.calloc(1, size
);
338 memcpy(retval
, ptr
, *old_size
);
341 * Mimick that a NULL pointer has been received, so
342 * memory allocation analysis based on the trace don't
343 * get confused by the address from the static
350 if (cur_alloc
.realloc
== NULL
) {
351 lookup_all_symbols();
352 if (cur_alloc
.realloc
== NULL
) {
353 fprintf(stderr
, "reallocwrap: unable to find realloc\n");
357 retval
= cur_alloc
.realloc(ptr
, size
);
359 if (URCU_TLS(malloc_nesting
) == 1) {
360 tracepoint(lttng_ust_libc
, realloc
,
361 ptr
, size
, retval
, LTTNG_UST_CALLER_IP());
363 URCU_TLS(malloc_nesting
)--;
367 void *memalign(size_t alignment
, size_t size
)
371 URCU_TLS(malloc_nesting
)++;
372 if (cur_alloc
.memalign
== NULL
) {
373 lookup_all_symbols();
374 if (cur_alloc
.memalign
== NULL
) {
375 fprintf(stderr
, "memalignwrap: unable to find memalign\n");
379 retval
= cur_alloc
.memalign(alignment
, size
);
380 if (URCU_TLS(malloc_nesting
) == 1) {
381 tracepoint(lttng_ust_libc
, memalign
,
382 alignment
, size
, retval
,
383 LTTNG_UST_CALLER_IP());
385 URCU_TLS(malloc_nesting
)--;
389 int posix_memalign(void **memptr
, size_t alignment
, size_t size
)
393 URCU_TLS(malloc_nesting
)++;
394 if (cur_alloc
.posix_memalign
== NULL
) {
395 lookup_all_symbols();
396 if (cur_alloc
.posix_memalign
== NULL
) {
397 fprintf(stderr
, "posix_memalignwrap: unable to find posix_memalign\n");
401 retval
= cur_alloc
.posix_memalign(memptr
, alignment
, size
);
402 if (URCU_TLS(malloc_nesting
) == 1) {
403 tracepoint(lttng_ust_libc
, posix_memalign
,
404 *memptr
, alignment
, size
,
405 retval
, LTTNG_UST_CALLER_IP());
407 URCU_TLS(malloc_nesting
)--;
412 void lttng_ust_fixup_malloc_nesting_tls(void)
414 asm volatile ("" : : "m" (URCU_TLS(malloc_nesting
)));
417 __attribute__((constructor
))
418 void lttng_ust_libc_wrapper_malloc_init(void)
420 /* Initialization already done */
421 if (cur_alloc
.calloc
) {
424 lttng_ust_fixup_malloc_nesting_tls();
426 * Ensure the allocator is in place before the process becomes
429 lookup_all_symbols();