2 * SPDX-License-Identifier: LGPL-2.1-or-later
4 * Copyright (C) 2009 Pierre-Marc Fournier
5 * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9 * Do _not_ define _LGPL_SOURCE because we don't want to create a
10 * circular dependency loop between this malloc wrapper, liburcu and
13 #include <ust-dlfcn.h>
14 #include <sys/types.h>
19 #include <urcu/system.h>
20 #include <urcu/uatomic.h>
21 #include <urcu/compiler.h>
22 #include <urcu/tls-compat.h>
23 #include <urcu/arch.h>
25 #include <lttng/ust-libc-wrapper.h>
27 #include <ust-helper.h>
28 #include "ust-compat.h"
30 #define TRACEPOINT_DEFINE
31 #define TRACEPOINT_CREATE_PROBES
32 #define TP_IP_PARAM ip
35 #define STATIC_CALLOC_LEN 4096
36 static char static_calloc_buf
[STATIC_CALLOC_LEN
];
37 static unsigned long static_calloc_buf_offset
;
39 struct alloc_functions
{
40 void *(*calloc
)(size_t nmemb
, size_t size
);
41 void *(*malloc
)(size_t size
);
42 void (*free
)(void *ptr
);
43 void *(*realloc
)(void *ptr
, size_t size
);
44 void *(*memalign
)(size_t alignment
, size_t size
);
45 int (*posix_memalign
)(void **memptr
, size_t alignment
, size_t size
);
49 struct alloc_functions cur_alloc
;
52 * Make sure our own use of the LTS compat layer will not cause infinite
53 * recursion by calling calloc.
57 void *static_calloc(size_t nmemb
, size_t size
);
60 * pthread mutex replacement for URCU tls compat layer.
62 static int ust_malloc_lock
;
65 void ust_malloc_spin_lock(pthread_mutex_t
*lock
)
66 __attribute__((unused
));
68 void ust_malloc_spin_lock(pthread_mutex_t
*lock
__attribute__((unused
)))
71 * The memory barrier within cmpxchg takes care of ordering
72 * memory accesses with respect to the start of the critical
75 while (uatomic_cmpxchg(&ust_malloc_lock
, 0, 1) != 0)
80 void ust_malloc_spin_unlock(pthread_mutex_t
*lock
)
81 __attribute__((unused
));
83 void ust_malloc_spin_unlock(pthread_mutex_t
*lock
__attribute__((unused
)))
86 * Ensure memory accesses within the critical section do not
90 uatomic_set(&ust_malloc_lock
, 0);
93 #define calloc static_calloc
94 #define pthread_mutex_lock ust_malloc_spin_lock
95 #define pthread_mutex_unlock ust_malloc_spin_unlock
96 static DEFINE_URCU_TLS(int, malloc_nesting
);
97 #undef pthread_mutex_unlock
98 #undef pthread_mutex_lock
102 * Static allocator to use when initially executing dlsym(). It keeps a
103 * size_t value of each object size prior to the object.
106 void *static_calloc_aligned(size_t nmemb
, size_t size
, size_t alignment
)
108 size_t prev_offset
, new_offset
, res_offset
, aligned_offset
;
110 if (nmemb
* size
== 0) {
115 * Protect static_calloc_buf_offset from concurrent updates
116 * using a cmpxchg loop rather than a mutex to remove a
117 * dependency on pthread. This will minimize the risk of bad
118 * interaction between mutex and malloc instrumentation.
120 res_offset
= CMM_LOAD_SHARED(static_calloc_buf_offset
);
122 prev_offset
= res_offset
;
123 aligned_offset
= LTTNG_UST_ALIGN(prev_offset
+ sizeof(size_t), alignment
);
124 new_offset
= aligned_offset
+ nmemb
* size
;
125 if (new_offset
> sizeof(static_calloc_buf
)) {
128 } while ((res_offset
= uatomic_cmpxchg(&static_calloc_buf_offset
,
129 prev_offset
, new_offset
)) != prev_offset
);
130 *(size_t *) &static_calloc_buf
[aligned_offset
- sizeof(size_t)] = size
;
131 return &static_calloc_buf
[aligned_offset
];
135 void *static_calloc(size_t nmemb
, size_t size
)
139 retval
= static_calloc_aligned(nmemb
, size
, 1);
144 void *static_malloc(size_t size
)
148 retval
= static_calloc_aligned(1, size
, 1);
153 void static_free(void *ptr
__attribute__((unused
)))
159 void *static_realloc(void *ptr
, size_t size
)
161 size_t *old_size
= NULL
;
170 old_size
= (size_t *) ptr
- 1;
171 if (size
<= *old_size
) {
172 /* We can re-use the old entry. */
178 /* We need to expand. Don't free previous memory location. */
179 retval
= static_calloc_aligned(1, size
, 1);
182 memcpy(retval
, ptr
, *old_size
);
188 void *static_memalign(size_t alignment
, size_t size
)
192 retval
= static_calloc_aligned(1, size
, alignment
);
197 int static_posix_memalign(void **memptr
, size_t alignment
, size_t size
)
201 /* Check for power of 2, larger than void *. */
202 if (alignment
& (alignment
- 1)
203 || alignment
< sizeof(void *)
207 ptr
= static_calloc_aligned(1, size
, alignment
);
214 void setup_static_allocator(void)
216 assert(cur_alloc
.calloc
== NULL
);
217 cur_alloc
.calloc
= static_calloc
;
218 assert(cur_alloc
.malloc
== NULL
);
219 cur_alloc
.malloc
= static_malloc
;
220 assert(cur_alloc
.free
== NULL
);
221 cur_alloc
.free
= static_free
;
222 assert(cur_alloc
.realloc
== NULL
);
223 cur_alloc
.realloc
= static_realloc
;
224 assert(cur_alloc
.memalign
== NULL
);
225 cur_alloc
.memalign
= static_memalign
;
226 assert(cur_alloc
.posix_memalign
== NULL
);
227 cur_alloc
.posix_memalign
= static_posix_memalign
;
231 void lookup_all_symbols(void)
233 struct alloc_functions af
;
236 * Temporarily redirect allocation functions to
237 * static_calloc_aligned, and free function to static_free
238 * (no-op), until the dlsym lookup has completed.
240 setup_static_allocator();
242 /* Perform the actual lookups */
243 af
.calloc
= dlsym(RTLD_NEXT
, "calloc");
244 af
.malloc
= dlsym(RTLD_NEXT
, "malloc");
245 af
.free
= dlsym(RTLD_NEXT
, "free");
246 af
.realloc
= dlsym(RTLD_NEXT
, "realloc");
247 af
.memalign
= dlsym(RTLD_NEXT
, "memalign");
248 af
.posix_memalign
= dlsym(RTLD_NEXT
, "posix_memalign");
250 /* Populate the new allocator functions */
251 memcpy(&cur_alloc
, &af
, sizeof(cur_alloc
));
254 void *malloc(size_t size
)
258 URCU_TLS(malloc_nesting
)++;
259 if (cur_alloc
.malloc
== NULL
) {
260 lookup_all_symbols();
261 if (cur_alloc
.malloc
== NULL
) {
262 fprintf(stderr
, "mallocwrap: unable to find malloc\n");
266 retval
= cur_alloc
.malloc(size
);
267 if (URCU_TLS(malloc_nesting
) == 1) {
268 tracepoint(lttng_ust_libc
, malloc
,
269 size
, retval
, LTTNG_UST_CALLER_IP());
271 URCU_TLS(malloc_nesting
)--;
277 URCU_TLS(malloc_nesting
)++;
279 * Check whether the memory was allocated with
280 * static_calloc_align, in which case there is nothing to free.
282 if (caa_unlikely((char *)ptr
>= static_calloc_buf
&&
283 (char *)ptr
< static_calloc_buf
+ STATIC_CALLOC_LEN
)) {
287 if (URCU_TLS(malloc_nesting
) == 1) {
288 tracepoint(lttng_ust_libc
, free
,
289 ptr
, LTTNG_UST_CALLER_IP());
292 if (cur_alloc
.free
== NULL
) {
293 lookup_all_symbols();
294 if (cur_alloc
.free
== NULL
) {
295 fprintf(stderr
, "mallocwrap: unable to find free\n");
301 URCU_TLS(malloc_nesting
)--;
304 void *calloc(size_t nmemb
, size_t size
)
308 URCU_TLS(malloc_nesting
)++;
309 if (cur_alloc
.calloc
== NULL
) {
310 lookup_all_symbols();
311 if (cur_alloc
.calloc
== NULL
) {
312 fprintf(stderr
, "callocwrap: unable to find calloc\n");
316 retval
= cur_alloc
.calloc(nmemb
, size
);
317 if (URCU_TLS(malloc_nesting
) == 1) {
318 tracepoint(lttng_ust_libc
, calloc
,
319 nmemb
, size
, retval
, LTTNG_UST_CALLER_IP());
321 URCU_TLS(malloc_nesting
)--;
325 void *realloc(void *ptr
, size_t size
)
329 URCU_TLS(malloc_nesting
)++;
331 * Check whether the memory was allocated with
332 * static_calloc_align, in which case there is nothing
333 * to free, and we need to copy the old data.
335 if (caa_unlikely((char *)ptr
>= static_calloc_buf
&&
336 (char *)ptr
< static_calloc_buf
+ STATIC_CALLOC_LEN
)) {
339 old_size
= (size_t *) ptr
- 1;
340 if (cur_alloc
.calloc
== NULL
) {
341 lookup_all_symbols();
342 if (cur_alloc
.calloc
== NULL
) {
343 fprintf(stderr
, "reallocwrap: unable to find calloc\n");
347 retval
= cur_alloc
.calloc(1, size
);
349 memcpy(retval
, ptr
, *old_size
);
352 * Mimick that a NULL pointer has been received, so
353 * memory allocation analysis based on the trace don't
354 * get confused by the address from the static
361 if (cur_alloc
.realloc
== NULL
) {
362 lookup_all_symbols();
363 if (cur_alloc
.realloc
== NULL
) {
364 fprintf(stderr
, "reallocwrap: unable to find realloc\n");
368 retval
= cur_alloc
.realloc(ptr
, size
);
370 if (URCU_TLS(malloc_nesting
) == 1) {
371 tracepoint(lttng_ust_libc
, realloc
,
372 ptr
, size
, retval
, LTTNG_UST_CALLER_IP());
374 URCU_TLS(malloc_nesting
)--;
378 void *memalign(size_t alignment
, size_t size
)
382 URCU_TLS(malloc_nesting
)++;
383 if (cur_alloc
.memalign
== NULL
) {
384 lookup_all_symbols();
385 if (cur_alloc
.memalign
== NULL
) {
386 fprintf(stderr
, "memalignwrap: unable to find memalign\n");
390 retval
= cur_alloc
.memalign(alignment
, size
);
391 if (URCU_TLS(malloc_nesting
) == 1) {
392 tracepoint(lttng_ust_libc
, memalign
,
393 alignment
, size
, retval
,
394 LTTNG_UST_CALLER_IP());
396 URCU_TLS(malloc_nesting
)--;
400 int posix_memalign(void **memptr
, size_t alignment
, size_t size
)
404 URCU_TLS(malloc_nesting
)++;
405 if (cur_alloc
.posix_memalign
== NULL
) {
406 lookup_all_symbols();
407 if (cur_alloc
.posix_memalign
== NULL
) {
408 fprintf(stderr
, "posix_memalignwrap: unable to find posix_memalign\n");
412 retval
= cur_alloc
.posix_memalign(memptr
, alignment
, size
);
413 if (URCU_TLS(malloc_nesting
) == 1) {
414 tracepoint(lttng_ust_libc
, posix_memalign
,
415 *memptr
, alignment
, size
,
416 retval
, LTTNG_UST_CALLER_IP());
418 URCU_TLS(malloc_nesting
)--;
423 void lttng_ust_fixup_malloc_nesting_tls(void)
425 asm volatile ("" : : "m" (URCU_TLS(malloc_nesting
)));
428 void lttng_ust_libc_wrapper_malloc_init(void)
430 /* Initialization already done */
431 if (cur_alloc
.calloc
) {
434 lttng_ust_fixup_malloc_nesting_tls();
436 * Ensure the allocator is in place before the process becomes
439 lookup_all_symbols();