src/lib/lttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-or-later
   3  *
   4  * Copyright (C) 2009 Pierre-Marc Fournier
   5  * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 /*
   9  * Do _not_ define _LGPL_SOURCE because we don't want to create a
  10  * circular dependency loop between this malloc wrapper, liburcu and
  11  * libc.
  12  */
  13
  14 /* Has to be included first to override dlfcn.h */
  15 #include <common/compat/dlfcn.h>
  16
  17 #include <sys/types.h>
  18 #include <stdio.h>
  19 #include <assert.h>
  20 #include <malloc.h>
  21
  22 #include <urcu/system.h>
  23 #include <urcu/uatomic.h>
  24 #include <urcu/compiler.h>
  25 #include <urcu/tls-compat.h>
  26 #include <urcu/arch.h>
  27
  28 #include <lttng/ust-libc-wrapper.h>
  29
  30 #include "common/macros.h"
  31 #include "common/align.h"
  32
  33 #define LTTNG_UST_TRACEPOINT_DEFINE
  34 #define LTTNG_UST_TRACEPOINT_CREATE_PROBES
  35 #define LTTNG_UST_TP_IP_PARAM ip
  36 #include "ust_libc.h"
  37
  38 #define STATIC_CALLOC_LEN 4096
  39 static char static_calloc_buf[STATIC_CALLOC_LEN];
  40 static unsigned long static_calloc_buf_offset;
  41
  42 struct alloc_functions {
  43         void *(*calloc)(size_t nmemb, size_t size);
  44         void *(*malloc)(size_t size);
  45         void (*free)(void *ptr);
  46         void *(*realloc)(void *ptr, size_t size);
  47         void *(*memalign)(size_t alignment, size_t size);
  48         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  49 };
  50
  51 static
  52 struct alloc_functions cur_alloc;
  53
  54 /*
  55  * Make sure our own use of the LTS compat layer will not cause infinite
  56  * recursion by calling calloc.
  57  */
  58
  59 static
  60 void *static_calloc(size_t nmemb, size_t size);
  61
  62 /*
  63  * pthread mutex replacement for URCU tls compat layer.
  64  */
  65 static int ust_malloc_lock;
  66
  67 static
  68 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  69         __attribute__((unused));
  70 static
  71 void ust_malloc_spin_lock(pthread_mutex_t *lock __attribute__((unused)))
  72 {
  73         /*
  74          * The memory barrier within cmpxchg takes care of ordering
  75          * memory accesses with respect to the start of the critical
  76          * section.
  77          */
  78         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  79                 caa_cpu_relax();
  80 }
  81
  82 static
  83 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  84         __attribute__((unused));
  85 static
  86 void ust_malloc_spin_unlock(pthread_mutex_t *lock __attribute__((unused)))
  87 {
  88         /*
  89          * Ensure memory accesses within the critical section do not
  90          * leak outside.
  91          */
  92         cmm_smp_mb();
  93         uatomic_set(&ust_malloc_lock, 0);
  94 }
  95
  96 #define calloc static_calloc
  97 #define pthread_mutex_lock ust_malloc_spin_lock
  98 #define pthread_mutex_unlock ust_malloc_spin_unlock
  99 static DEFINE_URCU_TLS(int, malloc_nesting);
 100 #undef pthread_mutex_unlock
 101 #undef pthread_mutex_lock
 102 #undef calloc
 103
 104 /*
 105  * Static allocator to use when initially executing dlsym(). It keeps a
 106  * size_t value of each object size prior to the object.
 107  */
 108 static
 109 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 110 {
 111         size_t prev_offset, new_offset, res_offset, aligned_offset;
 112
 113         if (nmemb * size == 0) {
 114                 return NULL;
 115         }
 116
 117         /*
 118          * Protect static_calloc_buf_offset from concurrent updates
 119          * using a cmpxchg loop rather than a mutex to remove a
 120          * dependency on pthread. This will minimize the risk of bad
 121          * interaction between mutex and malloc instrumentation.
 122          */
 123         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 124         do {
 125                 prev_offset = res_offset;
 126                 aligned_offset = LTTNG_UST_ALIGN(prev_offset + sizeof(size_t), alignment);
 127                 new_offset = aligned_offset + nmemb * size;
 128                 if (new_offset > sizeof(static_calloc_buf)) {
 129                         abort();
 130                 }
 131         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 132                         prev_offset, new_offset)) != prev_offset);
 133         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 134         return &static_calloc_buf[aligned_offset];
 135 }
 136
 137 static
 138 void *static_calloc(size_t nmemb, size_t size)
 139 {
 140         void *retval;
 141
 142         retval = static_calloc_aligned(nmemb, size, 1);
 143         return retval;
 144 }
 145
 146 static
 147 void *static_malloc(size_t size)
 148 {
 149         void *retval;
 150
 151         retval = static_calloc_aligned(1, size, 1);
 152         return retval;
 153 }
 154
 155 static
 156 void static_free(void *ptr __attribute__((unused)))
 157 {
 158         /* no-op. */
 159 }
 160
 161 static
 162 void *static_realloc(void *ptr, size_t size)
 163 {
 164         size_t *old_size = NULL;
 165         void *retval;
 166
 167         if (size == 0) {
 168                 retval = NULL;
 169                 goto end;
 170         }
 171
 172         if (ptr) {
 173                 old_size = (size_t *) ptr - 1;
 174                 if (size <= *old_size) {
 175                         /* We can re-use the old entry. */
 176                         *old_size = size;
 177                         retval = ptr;
 178                         goto end;
 179                 }
 180         }
 181         /* We need to expand. Don't free previous memory location. */
 182         retval = static_calloc_aligned(1, size, 1);
 183         assert(retval);
 184         if (ptr)
 185                 memcpy(retval, ptr, *old_size);
 186 end:
 187         return retval;
 188 }
 189
 190 static
 191 void *static_memalign(size_t alignment, size_t size)
 192 {
 193         void *retval;
 194
 195         retval = static_calloc_aligned(1, size, alignment);
 196         return retval;
 197 }
 198
 199 static
 200 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 201 {
 202         void *ptr;
 203
 204         /* Check for power of 2, larger than void *. */
 205         if (alignment & (alignment - 1)
 206                         || alignment < sizeof(void *)
 207                         || alignment == 0) {
 208                 goto end;
 209         }
 210         ptr = static_calloc_aligned(1, size, alignment);
 211         *memptr = ptr;
 212 end:
 213         return 0;
 214 }
 215
 216 static
 217 void setup_static_allocator(void)
 218 {
 219         assert(cur_alloc.calloc == NULL);
 220         cur_alloc.calloc = static_calloc;
 221         assert(cur_alloc.malloc == NULL);
 222         cur_alloc.malloc = static_malloc;
 223         assert(cur_alloc.free == NULL);
 224         cur_alloc.free = static_free;
 225         assert(cur_alloc.realloc == NULL);
 226         cur_alloc.realloc = static_realloc;
 227         assert(cur_alloc.memalign == NULL);
 228         cur_alloc.memalign = static_memalign;
 229         assert(cur_alloc.posix_memalign == NULL);
 230         cur_alloc.posix_memalign = static_posix_memalign;
 231 }
 232
 233 static
 234 void lookup_all_symbols(void)
 235 {
 236         struct alloc_functions af;
 237
 238         /*
 239          * Temporarily redirect allocation functions to
 240          * static_calloc_aligned, and free function to static_free
 241          * (no-op), until the dlsym lookup has completed.
 242          */
 243         setup_static_allocator();
 244
 245         /* Perform the actual lookups */
 246         af.calloc = dlsym(RTLD_NEXT, "calloc");
 247         af.malloc = dlsym(RTLD_NEXT, "malloc");
 248         af.free = dlsym(RTLD_NEXT, "free");
 249         af.realloc = dlsym(RTLD_NEXT, "realloc");
 250         af.memalign = dlsym(RTLD_NEXT, "memalign");
 251         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 252
 253         /* Populate the new allocator functions */
 254         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 255 }
 256
 257 void *malloc(size_t size)
 258 {
 259         void *retval;
 260
 261         URCU_TLS(malloc_nesting)++;
 262         if (cur_alloc.malloc == NULL) {
 263                 lookup_all_symbols();
 264                 if (cur_alloc.malloc == NULL) {
 265                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 266                         abort();
 267                 }
 268         }
 269         retval = cur_alloc.malloc(size);
 270         if (URCU_TLS(malloc_nesting) == 1) {
 271                 lttng_ust_tracepoint(lttng_ust_libc, malloc,
 272                         size, retval, LTTNG_UST_CALLER_IP());
 273         }
 274         URCU_TLS(malloc_nesting)--;
 275         return retval;
 276 }
 277
 278 void free(void *ptr)
 279 {
 280         URCU_TLS(malloc_nesting)++;
 281         /*
 282          * Check whether the memory was allocated with
 283          * static_calloc_align, in which case there is nothing to free.
 284          */
 285         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 286                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 287                 goto end;
 288         }
 289
 290         if (URCU_TLS(malloc_nesting) == 1) {
 291                 lttng_ust_tracepoint(lttng_ust_libc, free,
 292                         ptr, LTTNG_UST_CALLER_IP());
 293         }
 294
 295         if (cur_alloc.free == NULL) {
 296                 lookup_all_symbols();
 297                 if (cur_alloc.free == NULL) {
 298                         fprintf(stderr, "mallocwrap: unable to find free\n");
 299                         abort();
 300                 }
 301         }
 302         cur_alloc.free(ptr);
 303 end:
 304         URCU_TLS(malloc_nesting)--;
 305 }
 306
 307 void *calloc(size_t nmemb, size_t size)
 308 {
 309         void *retval;
 310
 311         URCU_TLS(malloc_nesting)++;
 312         if (cur_alloc.calloc == NULL) {
 313                 lookup_all_symbols();
 314                 if (cur_alloc.calloc == NULL) {
 315                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 316                         abort();
 317                 }
 318         }
 319         retval = cur_alloc.calloc(nmemb, size);
 320         if (URCU_TLS(malloc_nesting) == 1) {
 321                 lttng_ust_tracepoint(lttng_ust_libc, calloc,
 322                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 323         }
 324         URCU_TLS(malloc_nesting)--;
 325         return retval;
 326 }
 327
 328 void *realloc(void *ptr, size_t size)
 329 {
 330         void *retval;
 331
 332         URCU_TLS(malloc_nesting)++;
 333         /*
 334          * Check whether the memory was allocated with
 335          * static_calloc_align, in which case there is nothing
 336          * to free, and we need to copy the old data.
 337          */
 338         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 339                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 340                 size_t *old_size;
 341
 342                 old_size = (size_t *) ptr - 1;
 343                 if (cur_alloc.calloc == NULL) {
 344                         lookup_all_symbols();
 345                         if (cur_alloc.calloc == NULL) {
 346                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 347                                 abort();
 348                         }
 349                 }
 350                 retval = cur_alloc.calloc(1, size);
 351                 if (retval) {
 352                         memcpy(retval, ptr, *old_size);
 353                 }
 354                 /*
 355                  * Mimick that a NULL pointer has been received, so
 356                  * memory allocation analysis based on the trace don't
 357                  * get confused by the address from the static
 358                  * allocator.
 359                  */
 360                 ptr = NULL;
 361                 goto end;
 362         }
 363
 364         if (cur_alloc.realloc == NULL) {
 365                 lookup_all_symbols();
 366                 if (cur_alloc.realloc == NULL) {
 367                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 368                         abort();
 369                 }
 370         }
 371         retval = cur_alloc.realloc(ptr, size);
 372 end:
 373         if (URCU_TLS(malloc_nesting) == 1) {
 374                 lttng_ust_tracepoint(lttng_ust_libc, realloc,
 375                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 376         }
 377         URCU_TLS(malloc_nesting)--;
 378         return retval;
 379 }
 380
 381 void *memalign(size_t alignment, size_t size)
 382 {
 383         void *retval;
 384
 385         URCU_TLS(malloc_nesting)++;
 386         if (cur_alloc.memalign == NULL) {
 387                 lookup_all_symbols();
 388                 if (cur_alloc.memalign == NULL) {
 389                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 390                         abort();
 391                 }
 392         }
 393         retval = cur_alloc.memalign(alignment, size);
 394         if (URCU_TLS(malloc_nesting) == 1) {
 395                 lttng_ust_tracepoint(lttng_ust_libc, memalign,
 396                         alignment, size, retval,
 397                         LTTNG_UST_CALLER_IP());
 398         }
 399         URCU_TLS(malloc_nesting)--;
 400         return retval;
 401 }
 402
 403 int posix_memalign(void **memptr, size_t alignment, size_t size)
 404 {
 405         int retval;
 406
 407         URCU_TLS(malloc_nesting)++;
 408         if (cur_alloc.posix_memalign == NULL) {
 409                 lookup_all_symbols();
 410                 if (cur_alloc.posix_memalign == NULL) {
 411                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 412                         abort();
 413                 }
 414         }
 415         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 416         if (URCU_TLS(malloc_nesting) == 1) {
 417                 lttng_ust_tracepoint(lttng_ust_libc, posix_memalign,
 418                         *memptr, alignment, size,
 419                         retval, LTTNG_UST_CALLER_IP());
 420         }
 421         URCU_TLS(malloc_nesting)--;
 422         return retval;
 423 }
 424
 425 static
 426 void lttng_ust_fixup_malloc_nesting_tls(void)
 427 {
 428         asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
 429 }
 430
 431 void lttng_ust_libc_wrapper_malloc_ctor(void)
 432 {
 433         /* Initialization already done */
 434         if (cur_alloc.calloc) {
 435                 return;
 436         }
 437         lttng_ust_fixup_malloc_nesting_tls();
 438         /*
 439          * Ensure the allocator is in place before the process becomes
 440          * multithreaded.
 441          */
 442         lookup_all_symbols();
 443 }